diff --git a/numpy/core/SConscript b/numpy/core/SConscript index 0baea3d0c8ab..3a563e2e8069 100644 --- a/numpy/core/SConscript +++ b/numpy/core/SConscript @@ -439,6 +439,7 @@ if ENABLE_SEPARATE_COMPILATION: pjoin('src', 'multiarray', 'hashdescr.c'), pjoin('src', 'multiarray', 'arrayobject.c'), pjoin('src', 'multiarray', 'datetime.c'), + pjoin('src', 'multiarray', 'datetime_strings.c'), pjoin('src', 'multiarray', 'datetime_busday.c'), pjoin('src', 'multiarray', 'datetime_busdaycal.c'), pjoin('src', 'multiarray', 'numpyos.c'), diff --git a/numpy/core/arrayprint.py b/numpy/core/arrayprint.py index 556d4da04b22..567573af55b9 100644 --- a/numpy/core/arrayprint.py +++ b/numpy/core/arrayprint.py @@ -245,7 +245,7 @@ def _array2string(a, max_line_width, precision, suppress_small, separator=' ', 'complexfloat' : ComplexFormat(data, precision, suppress_small), 'longcomplexfloat' : LongComplexFormat(precision), - 'datetime' : DatetimeFormat(True, None, -1), + 'datetime' : DatetimeFormat(), 'timedelta' : TimedeltaFormat(data), 'numpystr' : repr, 'str' : str} @@ -698,16 +698,17 @@ def __call__(self, x): return r + i class DatetimeFormat(object): - def __init__(self, uselocaltime=True, overrideunit=None, tzoffset=-1): - self.local = uselocaltime + def __init__(self, overrideunit=None, + timezone='local', casting='same_kind'): + self.timezone = timezone self.unit = overrideunit - self.tzoffset = -1 + self.casting = casting def __call__(self, x): return "'%s'" % datetime_as_string(x, - local=self.local, unit=self.unit, - tzoffset=self.tzoffset) + timezone=self.timezone, + casting=self.casting) class TimedeltaFormat(object): def __init__(self, data): diff --git a/numpy/core/code_generators/genapi.py b/numpy/core/code_generators/genapi.py index 844aebff0af9..f69424d31eff 100644 --- a/numpy/core/code_generators/genapi.py +++ b/numpy/core/code_generators/genapi.py @@ -44,6 +44,7 @@ join('multiarray', 'conversion_utils.c'), join('multiarray', 'buffer.c'), join('multiarray', 'datetime.c'), + join('multiarray', 'datetime_strings.c'), join('multiarray', 'datetime_busday.c'), join('multiarray', 'datetime_busdaycal.c'), join('multiarray', 'nditer.c.src'), diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index d3e28fe93029..74f7855fd97a 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -85,7 +85,7 @@ enum NPY_TYPES { NPY_BOOL=0, NPY_NTYPES_ABI_COMPATIBLE=21 }; -#define NPY_METADATA_DTSTR "__frequency__" +#define NPY_METADATA_DTSTR "__timeunit__" /* basetype array priority */ #define NPY_PRIORITY 0.0 @@ -718,7 +718,9 @@ typedef struct { NPY_DATETIMEUNIT base; int num; /* - * 'den' is unused, kept here for ABI compatibility with 1.6. + * 'den' and 'events are unused, kept here for ABI + * compatibility with 1.6. + * * TODO: Remove for 2.0. */ int den; @@ -732,14 +734,12 @@ typedef struct { typedef struct { npy_int64 year; npy_int32 month, day, hour, min, sec, us, ps, as; - npy_int32 event; } npy_datetimestruct; /* TO BE REMOVED - NOT USED INTERNALLY. */ typedef struct { npy_int64 day; npy_int32 sec, us, ps, as; - npy_int32 event; } npy_timedeltastruct; /* TO BE REMOVED - NOT USED INTERNALLY. */ diff --git a/numpy/core/setup.py b/numpy/core/setup.py index 87e4e4f90f0d..c8d348410497 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -747,6 +747,7 @@ def get_mathlib_info(*args): join('src', 'multiarray', 'numpymemoryview.c'), join('src', 'multiarray', 'buffer.c'), join('src', 'multiarray', 'datetime.c'), + join('src', 'multiarray', 'datetime_strings.c'), join('src', 'multiarray', 'datetime_busday.c'), join('src', 'multiarray', 'datetime_busdaycal.c'), join('src', 'multiarray', 'numpyos.c'), diff --git a/numpy/core/src/multiarray/_datetime.h b/numpy/core/src/multiarray/_datetime.h index 8a9ffff7b885..3c216d85690f 100644 --- a/numpy/core/src/multiarray/_datetime.h +++ b/numpy/core/src/multiarray/_datetime.h @@ -1,9 +1,25 @@ #ifndef _NPY_PRIVATE__DATETIME_H_ #define _NPY_PRIVATE__DATETIME_H_ +NPY_NO_EXPORT char *_datetime_strings[NPY_DATETIME_NUMUNITS]; + +NPY_NO_EXPORT int _days_per_month_table[2][12]; + NPY_NO_EXPORT void numpy_pydatetime_import(); +/* + * Returns 1 if the given year is a leap year, 0 otherwise. + */ +NPY_NO_EXPORT int +is_leapyear(npy_int64 year); + +/* + * Calculates the days offset from the 1970 epoch. + */ +NPY_NO_EXPORT npy_int64 +get_datetimestruct_days(const npy_datetimestruct *dts); + /* * Creates a datetime or timedelta dtype using a copy of the provided metadata. */ @@ -99,10 +115,48 @@ convert_datetime_divisor_to_multiple(PyArray_DatetimeMetaData *meta, */ NPY_NO_EXPORT npy_bool datetime_metadata_divides( - PyArray_Descr *dividend, - PyArray_Descr *divisor, + PyArray_DatetimeMetaData *dividend, + PyArray_DatetimeMetaData *divisor, int strict_with_nonlinear_units); +/* + * This provides the casting rules for the DATETIME data type units. + * + * Notably, there is a barrier between 'date units' and 'time units' + * for all but 'unsafe' casting. + */ +NPY_NO_EXPORT npy_bool +can_cast_datetime64_units(NPY_DATETIMEUNIT src_unit, + NPY_DATETIMEUNIT dst_unit, + NPY_CASTING casting); + +/* + * This provides the casting rules for the DATETIME data type metadata. + */ +NPY_NO_EXPORT npy_bool +can_cast_datetime64_metadata(PyArray_DatetimeMetaData *src_meta, + PyArray_DatetimeMetaData *dst_meta, + NPY_CASTING casting); + +/* + * This provides the casting rules for the TIMEDELTA data type units. + * + * Notably, there is a barrier between the nonlinear years and + * months units, and all the other units. + */ +NPY_NO_EXPORT npy_bool +can_cast_timedelta64_units(NPY_DATETIMEUNIT src_unit, + NPY_DATETIMEUNIT dst_unit, + NPY_CASTING casting); + +/* + * This provides the casting rules for the TIMEDELTA data type metadata. + */ +NPY_NO_EXPORT npy_bool +can_cast_timedelta64_metadata(PyArray_DatetimeMetaData *src_meta, + PyArray_DatetimeMetaData *dst_meta, + NPY_CASTING casting); + /* * Computes the GCD of the two date-time metadata values. Raises * an exception if there is no reasonable GCD, such as with @@ -119,11 +173,7 @@ compute_datetime_metadata_greatest_common_divisor_capsule( /* * Computes the conversion factor to convert data with 'src_meta' metadata - * into data with 'dst_meta' metadata, not taking into account the events. - * - * To convert a npy_datetime or npy_timedelta, first the event number needs - * to be divided away, then it needs to be scaled by num/denom, and - * finally the event number can be added back in. + * into data with 'dst_meta' metadata. * * If overflow occurs, both out_num and out_denom are set to 0, but * no error is set. @@ -156,6 +206,13 @@ convert_datetime_metadata_tuple_to_datetime_metadata(PyObject *tuple, NPY_NO_EXPORT PyObject * convert_datetime_metadata_tuple_to_metacobj(PyObject *tuple); +/* + * Gets a tzoffset in minutes by calling the fromutc() function on + * the Python datetime.tzinfo object. + */ +NPY_NO_EXPORT int +get_tzoffset_from_pytzinfo(PyObject *timezone, npy_datetimestruct *dts); + /* * Converts an input object into datetime metadata. The input * may be either a string or a tuple. @@ -170,7 +227,7 @@ convert_pyobject_to_datetime_metadata(PyObject *obj, * 'ret' is a PyUString containing the datetime string, and this * function appends the metadata string to it. * - * If 'skip_brackets' is true, skips the '[]' when events == 1. + * If 'skip_brackets' is true, skips the '[]'. * * This function steals the reference 'ret' */ @@ -179,77 +236,6 @@ append_metastr_to_string(PyArray_DatetimeMetaData *meta, int skip_brackets, PyObject *ret); -/* - * Provides a string length to use for converting datetime - * objects with the given local and unit settings. - */ -NPY_NO_EXPORT int -get_datetime_iso_8601_strlen(int local, NPY_DATETIMEUNIT base); - -/* - * Parses (almost) standard ISO 8601 date strings. The differences are: - * - * + After the date and time, may place a ' ' followed by an event number. - * + The date "20100312" is parsed as the year 20100312, not as - * equivalent to "2010-03-12". The '-' in the dates are not optional. - * + Only seconds may have a decimal point, with up to 18 digits after it - * (maximum attoseconds precision). - * + Either a 'T' as in ISO 8601 or a ' ' may be used to separate - * the date and the time. Both are treated equivalently. - * + Doesn't (yet) handle the "YYYY-DDD" or "YYYY-Www" formats. - * + Doesn't handle leap seconds (seconds value has 60 in these cases). - * + Doesn't handle 24:00:00 as synonym for midnight (00:00:00) tomorrow - * + Accepts special values "NaT" (not a time), "Today", (current - * day according to local time) and "Now" (current time in UTC). - * - * 'str' must be a NULL-terminated string, and 'len' must be its length. - * 'unit' should contain -1 if the unit is unknown, or the unit - * which will be used if it is. - * - * 'out' gets filled with the parsed date-time. - * 'out_local' gets set to 1 if the parsed time was in local time, - * to 0 otherwise. The values 'now' and 'today' don't get counted - * as local, and neither do UTC +/-#### timezone offsets, because - * they aren't using the computer's local timezone offset. - * 'out_bestunit' gives a suggested unit based on the amount of - * resolution provided in the string, or -1 for NaT. - * 'out_special' gets set to 1 if the parsed time was 'today', - * 'now', or ''/'NaT'. For 'today', the unit recommended is - * 'D', for 'now', the unit recommended is 's', and for 'NaT' - * the unit recommended is 'Y'. - * - * - * Returns 0 on success, -1 on failure. - */ -NPY_NO_EXPORT int -parse_iso_8601_date(char *str, int len, - NPY_DATETIMEUNIT unit, - npy_datetimestruct *out, - npy_bool *out_local, - NPY_DATETIMEUNIT *out_bestunit, - npy_bool *out_special); - -/* - * Converts an npy_datetimestruct to an (almost) ISO 8601 - * NULL-terminated string. - * - * If 'local' is non-zero, it produces a string in local time with - * a +-#### timezone offset, otherwise it uses timezone Z (UTC). - * - * 'base' restricts the output to that unit. Set 'base' to - * -1 to auto-detect a base after which all the values are zero. - * - * 'tzoffset' is used if 'local' is enabled, and 'tzoffset' is - * set to a value other than -1. This is a manual override for - * the local time zone to use, as an offset in minutes. - * - * Returns 0 on success, -1 on failure (for example if the output - * string was too short). - */ -NPY_NO_EXPORT int -make_iso_8601_date(npy_datetimestruct *dts, char *outstr, int outlen, - int local, NPY_DATETIMEUNIT base, int tzoffset); - /* * Tests for and converts a Python datetime.datetime or datetime.date * object into a NumPy npy_datetimestruct. @@ -257,12 +243,16 @@ make_iso_8601_date(npy_datetimestruct *dts, char *outstr, int outlen, * 'out_bestunit' gives a suggested unit based on whether the object * was a datetime.date or datetime.datetime object. * + * If 'apply_tzinfo' is 1, this function uses the tzinfo to convert + * to UTC time, otherwise it returns the struct with the local time. + * * Returns -1 on error, 0 on success, and 1 (with no error set) * if obj doesn't have the neeeded date or datetime attributes. */ NPY_NO_EXPORT int convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, - NPY_DATETIMEUNIT *out_bestunit); + NPY_DATETIMEUNIT *out_bestunit, + int apply_tzinfo); /* * Converts a PyObject * into a datetime, in any of the forms supported. @@ -271,11 +261,17 @@ convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, * to -1, and this function will populate meta with either default * values or values from the input object. * + * The 'casting' parameter is used to control what kinds of inputs + * are accepted, and what happens. For example, with 'unsafe' casting, + * unrecognized inputs are converted to 'NaT' instead of throwing an error, + * while with 'safe' casting an error will be thrown if any precision + * from the input will be thrown away. + * * Returns -1 on error, 0 on success. */ NPY_NO_EXPORT int convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, - npy_datetime *out); + NPY_CASTING casting, npy_datetime *out); /* * Converts a PyObject * into a timedelta, in any of the forms supported @@ -284,12 +280,17 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, * to -1, and this function will populate meta with either default * values or values from the input object. * + * The 'casting' parameter is used to control what kinds of inputs + * are accepted, and what happens. For example, with 'unsafe' casting, + * unrecognized inputs are converted to 'NaT' instead of throwing an error, + * while with 'safe' casting an error will be thrown if any precision + * from the input will be thrown away. + * * Returns -1 on error, 0 on success. */ NPY_NO_EXPORT int convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj, - npy_timedelta *out); - + NPY_CASTING casting, npy_timedelta *out); /* * Converts a datetime into a PyObject *. diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src index e4dab1f7e6b7..b27c7e43fb0e 100644 --- a/numpy/core/src/multiarray/arraytypes.c.src +++ b/numpy/core/src/multiarray/arraytypes.c.src @@ -817,7 +817,8 @@ DATETIME_setitem(PyObject *op, char *ov, PyArrayObject *ap) { } /* Convert the object into a NumPy datetime */ - if (convert_pyobject_to_datetime(meta, op, &temp) < 0) { + if (convert_pyobject_to_datetime(meta, op, + NPY_SAME_KIND_CASTING, &temp) < 0) { return -1; } @@ -845,7 +846,8 @@ TIMEDELTA_setitem(PyObject *op, char *ov, PyArrayObject *ap) { } /* Convert the object into a NumPy datetime */ - if (convert_pyobject_to_timedelta(meta, op, &temp) < 0) { + if (convert_pyobject_to_timedelta(meta, op, + NPY_SAME_KIND_CASTING, &temp) < 0) { return -1; } @@ -3436,7 +3438,6 @@ _init_datetime_descr(PyArray_Descr *descr) dt_data = PyArray_malloc(sizeof(PyArray_DatetimeMetaData)); dt_data->base = NPY_DATETIME_DEFAULTUNIT; dt_data->num = 1; - dt_data->events = 1; /* FIXME * There is no error check here and no way to indicate an error diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index 28846462d0a6..2b0a89d2af90 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -309,7 +309,7 @@ _array_typedescr_fromstr(char *str) switch (typechar) { case 'b': if (size == sizeof(Bool)) { - type_num = PyArray_BOOL; + type_num = NPY_BOOL; } else { PyErr_SetString(PyExc_ValueError, msg); @@ -318,22 +318,22 @@ _array_typedescr_fromstr(char *str) break; case 'u': if (size == sizeof(uintp)) { - type_num = PyArray_UINTP; + type_num = NPY_UINTP; } else if (size == sizeof(char)) { - type_num = PyArray_UBYTE; + type_num = NPY_UBYTE; } else if (size == sizeof(short)) { - type_num = PyArray_USHORT; + type_num = NPY_USHORT; } else if (size == sizeof(ulong)) { - type_num = PyArray_ULONG; + type_num = NPY_ULONG; } else if (size == sizeof(int)) { - type_num = PyArray_UINT; + type_num = NPY_UINT; } else if (size == sizeof(ulonglong)) { - type_num = PyArray_ULONGLONG; + type_num = NPY_ULONGLONG; } else { PyErr_SetString(PyExc_ValueError, msg); @@ -342,22 +342,22 @@ _array_typedescr_fromstr(char *str) break; case 'i': if (size == sizeof(intp)) { - type_num = PyArray_INTP; + type_num = NPY_INTP; } else if (size == sizeof(char)) { - type_num = PyArray_BYTE; + type_num = NPY_BYTE; } else if (size == sizeof(short)) { - type_num = PyArray_SHORT; + type_num = NPY_SHORT; } else if (size == sizeof(long)) { - type_num = PyArray_LONG; + type_num = NPY_LONG; } else if (size == sizeof(int)) { - type_num = PyArray_INT; + type_num = NPY_INT; } else if (size == sizeof(longlong)) { - type_num = PyArray_LONGLONG; + type_num = NPY_LONGLONG; } else { PyErr_SetString(PyExc_ValueError, msg); @@ -366,13 +366,13 @@ _array_typedescr_fromstr(char *str) break; case 'f': if (size == sizeof(float)) { - type_num = PyArray_FLOAT; + type_num = NPY_FLOAT; } else if (size == sizeof(double)) { - type_num = PyArray_DOUBLE; + type_num = NPY_DOUBLE; } else if (size == sizeof(longdouble)) { - type_num = PyArray_LONGDOUBLE; + type_num = NPY_LONGDOUBLE; } else { PyErr_SetString(PyExc_ValueError, msg); @@ -381,13 +381,13 @@ _array_typedescr_fromstr(char *str) break; case 'c': if (size == sizeof(float)*2) { - type_num = PyArray_CFLOAT; + type_num = NPY_CFLOAT; } else if (size == sizeof(double)*2) { - type_num = PyArray_CDOUBLE; + type_num = NPY_CDOUBLE; } else if (size == sizeof(longdouble)*2) { - type_num = PyArray_CLONGDOUBLE; + type_num = NPY_CLONGDOUBLE; } else { PyErr_SetString(PyExc_ValueError, msg); @@ -396,22 +396,22 @@ _array_typedescr_fromstr(char *str) break; case 'O': if (size == sizeof(PyObject *)) { - type_num = PyArray_OBJECT; + type_num = NPY_OBJECT; } else { PyErr_SetString(PyExc_ValueError, msg); return NULL; } break; - case PyArray_STRINGLTR: - type_num = PyArray_STRING; + case NPY_STRINGLTR: + type_num = NPY_STRING; break; - case PyArray_UNICODELTR: - type_num = PyArray_UNICODE; + case NPY_UNICODELTR: + type_num = NPY_UNICODE; size <<= 2; break; case 'V': - type_num = PyArray_VOID; + type_num = NPY_VOID; break; default: PyErr_SetString(PyExc_ValueError, msg); diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c index 9e0b93a56fbf..acaa84185a2f 100644 --- a/numpy/core/src/multiarray/convert_datatype.c +++ b/numpy/core/src/multiarray/convert_datatype.c @@ -17,6 +17,7 @@ #include "convert_datatype.h" #include "_datetime.h" +#include "datetime_strings.h" /*NUMPY_API * For backward compatibility @@ -31,30 +32,11 @@ NPY_NO_EXPORT PyObject * PyArray_CastToType(PyArrayObject *arr, PyArray_Descr *dtype, int fortran) { PyObject *out; - PyArray_Descr *arr_dtype; - arr_dtype = PyArray_DESCR(arr); - - if (dtype->elsize == 0) { - PyArray_DESCR_REPLACE(dtype); - if (dtype == NULL) { - return NULL; - } - - if (arr_dtype->type_num == dtype->type_num) { - dtype->elsize = arr_dtype->elsize; - } - else if (arr_dtype->type_num == NPY_STRING && - dtype->type_num == NPY_UNICODE) { - dtype->elsize = arr_dtype->elsize * 4; - } - else if (arr_dtype->type_num == NPY_UNICODE && - dtype->type_num == NPY_STRING) { - dtype->elsize = arr_dtype->elsize / 4; - } - else if (dtype->type_num == NPY_VOID) { - dtype->elsize = arr_dtype->elsize; - } + /* If the requested dtype is flexible, adapt it */ + PyArray_AdaptFlexibleDType((PyObject *)arr, PyArray_DESCR(arr), &dtype); + if (dtype == NULL) { + return NULL; } out = PyArray_NewFromDescr(Py_TYPE(arr), dtype, @@ -136,6 +118,170 @@ PyArray_GetCastFunc(PyArray_Descr *descr, int type_num) return NULL; } +/* + * This function calls Py_DECREF on flex_dtype, and replaces it with + * a new dtype that has been adapted based on the values in data_dtype + * and data_obj. If the flex_dtype is not flexible, it leaves it as is. + * + * Usually, if data_obj is not an array, dtype should be the result + * given by the PyArray_GetArrayParamsFromObject function. + * + * The data_obj may be NULL if just a dtype is is known for the source. + * + * If *flex_dtype is NULL, returns immediately, without setting an + * exception. This basically assumes an error was already set previously. + * + * The current flexible dtypes include NPY_STRING, NPY_UNICODE, NPY_VOID, + * and NPY_DATETIME with generic units. + */ +NPY_NO_EXPORT void +PyArray_AdaptFlexibleDType(PyObject *data_obj, PyArray_Descr *data_dtype, + PyArray_Descr **flex_dtype) +{ + PyArray_DatetimeMetaData *meta; + int flex_type_num; + + if (*flex_dtype == NULL) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, + "NumPy AdaptFlexibleDType was called with NULL flex_dtype " + "but no error set"); + } + return; + } + + flex_type_num = (*flex_dtype)->type_num; + + /* Flexible types with expandable size */ + if ((*flex_dtype)->elsize == 0) { + /* First replace the flex dtype */ + PyArray_DESCR_REPLACE(*flex_dtype); + if (*flex_dtype == NULL) { + return; + } + + if (data_dtype->type_num == flex_type_num || + flex_type_num == NPY_VOID) { + (*flex_dtype)->elsize = data_dtype->elsize; + } + else { + npy_intp size = 8; + + /* + * Get a string-size estimate of the input. These + * are generallly the size needed, rounded up to + * a multiple of eight. + */ + switch (data_dtype->type_num) { + case NPY_BOOL: + size = 8; + break; + case NPY_UBYTE: + size = 8; + break; + case NPY_BYTE: + size = 8; + break; + case NPY_USHORT: + size = 8; + break; + case NPY_SHORT: + size = 8; + break; + case NPY_UINT: + size = 16; + break; + case NPY_INT: + size = 16; + break; + case NPY_ULONG: + size = 24; + break; + case NPY_LONG: + size = 24; + break; + case NPY_ULONGLONG: + size = 24; + break; + case NPY_LONGLONG: + size = 24; + break; + case NPY_HALF: + case NPY_FLOAT: + case NPY_DOUBLE: + case NPY_LONGDOUBLE: + size = 32; + break; + case NPY_CFLOAT: + case NPY_CDOUBLE: + case NPY_CLONGDOUBLE: + size = 64; + break; + case NPY_OBJECT: + size = 64; + break; + case NPY_STRING: + case NPY_VOID: + size = data_dtype->elsize; + break; + case NPY_UNICODE: + size = data_dtype->elsize / 4; + break; + case NPY_DATETIME: + meta = get_datetime_metadata_from_dtype(data_dtype); + if (meta == NULL) { + Py_DECREF(*flex_dtype); + *flex_dtype = NULL; + return; + } + size = get_datetime_iso_8601_strlen(0, meta->base); + break; + case NPY_TIMEDELTA: + size = 21; + break; + } + + if (flex_type_num == NPY_STRING) { + (*flex_dtype)->elsize = size; + } + else if (flex_type_num == NPY_UNICODE) { + (*flex_dtype)->elsize = size * 4; + } + } + } + /* Flexible type with generic time unit that adapts */ + else if (flex_type_num == NPY_DATETIME || + flex_type_num == NPY_TIMEDELTA) { + meta = get_datetime_metadata_from_dtype(*flex_dtype); + if (meta == NULL) { + Py_DECREF(*flex_dtype); + *flex_dtype = NULL; + return; + } + + if (meta->base == NPY_FR_GENERIC) { + if (data_dtype->type_num == NPY_DATETIME || + data_dtype->type_num == NPY_TIMEDELTA) { + meta = get_datetime_metadata_from_dtype(data_dtype); + if (meta == NULL) { + Py_DECREF(*flex_dtype); + *flex_dtype = NULL; + return; + } + + Py_DECREF(*flex_dtype); + *flex_dtype = create_datetime_dtype(flex_type_num, meta); + } + else if (data_obj != NULL) { + /* Detect the unit from the input's data */ + Py_DECREF(*flex_dtype); + *flex_dtype = find_object_datetime_type(data_obj, + flex_type_num); + } + } + } +} + /* * Must be broadcastable. * This code is very similar to PyArray_CopyInto/PyArray_MoveInto @@ -184,21 +330,21 @@ PyArray_CanCastSafely(int fromtype, int totype) } /* Special-cases for some types */ switch (fromtype) { - case PyArray_DATETIME: - case PyArray_TIMEDELTA: - case PyArray_OBJECT: - case PyArray_VOID: + case NPY_DATETIME: + case NPY_TIMEDELTA: + case NPY_OBJECT: + case NPY_VOID: return 0; - case PyArray_BOOL: + case NPY_BOOL: return 1; } switch (totype) { - case PyArray_BOOL: - case PyArray_DATETIME: - case PyArray_TIMEDELTA: + case NPY_BOOL: + case NPY_DATETIME: + case NPY_TIMEDELTA: return 0; - case PyArray_OBJECT: - case PyArray_VOID: + case NPY_OBJECT: + case NPY_VOID: return 1; } @@ -210,7 +356,7 @@ PyArray_CanCastSafely(int fromtype, int totype) if (from->f->cancastto) { int *curtype = from->f->cancastto; - while (*curtype != PyArray_NOTYPE) { + while (*curtype != NPY_NOTYPE) { if (*curtype++ == totype) { return 1; } @@ -228,23 +374,23 @@ PyArray_CanCastSafely(int fromtype, int totype) NPY_NO_EXPORT npy_bool PyArray_CanCastTo(PyArray_Descr *from, PyArray_Descr *to) { - int fromtype=from->type_num; - int totype=to->type_num; + int from_type_num = from->type_num; + int to_type_num = to->type_num; npy_bool ret; - ret = (npy_bool) PyArray_CanCastSafely(fromtype, totype); + ret = (npy_bool) PyArray_CanCastSafely(from_type_num, to_type_num); if (ret) { /* Check String and Unicode more closely */ - if (fromtype == NPY_STRING) { - if (totype == NPY_STRING) { + if (from_type_num == NPY_STRING) { + if (to_type_num == NPY_STRING) { ret = (from->elsize <= to->elsize); } - else if (totype == NPY_UNICODE) { + else if (to_type_num == NPY_UNICODE) { ret = (from->elsize << 2 <= to->elsize); } } - else if (fromtype == NPY_UNICODE) { - if (totype == NPY_UNICODE) { + else if (from_type_num == NPY_UNICODE) { + if (to_type_num == NPY_UNICODE) { ret = (from->elsize <= to->elsize); } } @@ -252,14 +398,41 @@ PyArray_CanCastTo(PyArray_Descr *from, PyArray_Descr *to) * For datetime/timedelta, only treat casts moving towards * more precision as safe. */ - else if (fromtype == NPY_DATETIME && totype == NPY_DATETIME) { - return datetime_metadata_divides(from, to, 0); + else if (from_type_num == NPY_DATETIME && to_type_num == NPY_DATETIME) { + PyArray_DatetimeMetaData *meta1, *meta2; + meta1 = get_datetime_metadata_from_dtype(from); + if (meta1 == NULL) { + PyErr_Clear(); + return 0; + } + meta2 = get_datetime_metadata_from_dtype(to); + if (meta2 == NULL) { + PyErr_Clear(); + return 0; + } + + return can_cast_datetime64_metadata(meta1, meta2, + NPY_SAFE_CASTING); } - else if (fromtype == NPY_TIMEDELTA && totype == NPY_TIMEDELTA) { - return datetime_metadata_divides(from, to, 1); + else if (from_type_num == NPY_TIMEDELTA && + to_type_num == NPY_TIMEDELTA) { + PyArray_DatetimeMetaData *meta1, *meta2; + meta1 = get_datetime_metadata_from_dtype(from); + if (meta1 == NULL) { + PyErr_Clear(); + return 0; + } + meta2 = get_datetime_metadata_from_dtype(to); + if (meta2 == NULL) { + PyErr_Clear(); + return 0; + } + + return can_cast_timedelta64_metadata(meta1, meta2, + NPY_SAFE_CASTING); } /* - * TODO: If totype is STRING or unicode + * TODO: If to_type_num is STRING or unicode * see if the length is long enough to hold the * stringified value of the object. */ @@ -374,22 +547,50 @@ PyArray_CanCastTypeTo(PyArray_Descr *from, PyArray_Descr *to, } switch (from->type_num) { - case NPY_DATETIME: - case NPY_TIMEDELTA: - switch (casting) { - case NPY_NO_CASTING: - return PyArray_ISNBO(from->byteorder) == - PyArray_ISNBO(to->byteorder) && - has_equivalent_datetime_metadata(from, to); - case NPY_EQUIV_CASTING: - return has_equivalent_datetime_metadata(from, to); - case NPY_SAFE_CASTING: - return datetime_metadata_divides(from, to, - from->type_num == NPY_TIMEDELTA); - default: - return 1; + case NPY_DATETIME: { + PyArray_DatetimeMetaData *meta1, *meta2; + meta1 = get_datetime_metadata_from_dtype(from); + if (meta1 == NULL) { + PyErr_Clear(); + return 0; } - break; + meta2 = get_datetime_metadata_from_dtype(to); + if (meta2 == NULL) { + PyErr_Clear(); + return 0; + } + + if (casting == NPY_NO_CASTING) { + return PyArray_ISNBO(from->byteorder) == + PyArray_ISNBO(to->byteorder) && + can_cast_datetime64_metadata(meta1, meta2, casting); + } + else { + return can_cast_datetime64_metadata(meta1, meta2, casting); + } + } + case NPY_TIMEDELTA: { + PyArray_DatetimeMetaData *meta1, *meta2; + meta1 = get_datetime_metadata_from_dtype(from); + if (meta1 == NULL) { + PyErr_Clear(); + return 0; + } + meta2 = get_datetime_metadata_from_dtype(to); + if (meta2 == NULL) { + PyErr_Clear(); + return 0; + } + + if (casting == NPY_NO_CASTING) { + return PyArray_ISNBO(from->byteorder) == + PyArray_ISNBO(to->byteorder) && + can_cast_timedelta64_metadata(meta1, meta2, casting); + } + else { + return can_cast_timedelta64_metadata(meta1, meta2, casting); + } + } default: switch (casting) { case NPY_NO_CASTING: diff --git a/numpy/core/src/multiarray/convert_datatype.h b/numpy/core/src/multiarray/convert_datatype.h index 844cce0c96fa..71001b1c470d 100644 --- a/numpy/core/src/multiarray/convert_datatype.h +++ b/numpy/core/src/multiarray/convert_datatype.h @@ -13,4 +13,16 @@ PyArray_ConvertToCommonType(PyObject *op, int *retn); NPY_NO_EXPORT int PyArray_ValidType(int type); +/* + * This function calls Py_DECREF on flex_dtype, and replaces it with + * a new dtype that has been adapted based on the values in data_dtype + * and data_obj. If the flex_dtype is not flexible, it leaves it as is. + * + * The current flexible dtypes include NPY_STRING, NPY_UNICODE, NPY_VOID, + * and NPY_DATETIME with generic units. + */ +NPY_NO_EXPORT void +PyArray_AdaptFlexibleDType(PyObject *data_obj, PyArray_Descr *data_dtype, + PyArray_Descr **flex_dtype); + #endif diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 8d4a7430b038..2c68e026bf12 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -15,11 +15,13 @@ #include "common.h" #include "ctors.h" +#include "convert_datatype.h" #include "shape.h" #include "buffer.h" #include "numpymemoryview.h" #include "lowlevel_strided_loops.h" #include "_datetime.h" +#include "datetime_strings.h" /* * Reading from a file or a string. @@ -1653,72 +1655,14 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, 0, &dtype, &ndim, dims, &arr, context) < 0) { Py_XDECREF(newtype); - ret = NULL; return NULL; } - /* If the requested dtype is flexible, adjust its size */ - if (newtype != NULL && newtype->elsize == 0) { - PyArray_DESCR_REPLACE(newtype); - if (newtype == NULL) { - ret = NULL; - return NULL; - } - if (arr != NULL) { - dtype = PyArray_DESCR(arr); - } - - if (newtype->type_num == dtype->type_num) { - newtype->elsize = dtype->elsize; - } - else { - switch(newtype->type_num) { - case NPY_STRING: - if (dtype->type_num == NPY_UNICODE) { - newtype->elsize = dtype->elsize >> 2; - } - else { - newtype->elsize = dtype->elsize; - } - break; - case NPY_UNICODE: - newtype->elsize = dtype->elsize << 2; - break; - case NPY_VOID: - newtype->elsize = dtype->elsize; - break; - } - } - } - /* - * Treat datetime generic units with the same idea as flexible strings. - * - * Flexible strings, for example the dtype 'str', use size zero as a - * signal indicating that they represent a "generic string type" instead - * of a string type with the size already baked in. The generic unit - * plays the same role, indicating that it's a "generic datetime type", - * and the actual unit should be filled in when needed just like the - * actual string size should be filled in when needed. - */ - else if (newtype != NULL && newtype->type_num == NPY_DATETIME) { - PyArray_DatetimeMetaData *meta = - get_datetime_metadata_from_dtype(newtype); - if (meta == NULL) { - Py_DECREF(newtype); - return NULL; - } - - if (meta->base == NPY_FR_GENERIC) { - /* Detect the unit from the input's data */ - PyArray_Descr *dtype = find_object_datetime_type(op, - newtype->type_num); - if (dtype == NULL) { - Py_DECREF(newtype); - return NULL; - } - Py_DECREF(newtype); - newtype = dtype; - } + /* If the requested dtype is flexible, adapt it */ + if (newtype != NULL) { + PyArray_AdaptFlexibleDType(op, + (dtype == NULL) ? PyArray_DESCR(arr) : dtype, + &newtype); } /* If we got dimensions and dtype instead of an array */ diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 16ac5c4fbf38..cdf99546f31e 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -1,5 +1,5 @@ /* - * This file implements core functionality for NumPy datetime + * This file implements core functionality for NumPy datetime. * * Written by Mark Wiebe (mwwiebe@gmail.com) * Copyright (c) 2011 by Enthought, Inc. @@ -20,7 +20,9 @@ #include "numpy/npy_3kcompat.h" #include "numpy/arrayscalars.h" +#include "methods.h" #include "_datetime.h" +#include "datetime_strings.h" /* * Imports the PyDateTime functions so we can create these objects. @@ -32,11 +34,8 @@ numpy_pydatetime_import() PyDateTime_IMPORT; } -static int -is_leapyear(npy_int64 year); - /* Exported as DATETIMEUNITS in multiarraymodule.c */ -NPY_NO_EXPORT char *_datetime_strings[] = { +NPY_NO_EXPORT char *_datetime_strings[NPY_DATETIME_NUMUNITS] = { NPY_STR_Y, NPY_STR_M, NPY_STR_W, @@ -54,12 +53,15 @@ NPY_NO_EXPORT char *_datetime_strings[] = { }; /* Days per month, regular year and leap year */ -static int days_in_month[2][12] = { +NPY_NO_EXPORT int _days_per_month_table[2][12] = { { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }, { 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 } }; -static int +/* + * Returns 1 if the given year is a leap year, 0 otherwise. + */ +NPY_NO_EXPORT int is_leapyear(npy_int64 year) { return (year & 0x3) == 0 && /* year % 4 == 0 */ @@ -70,7 +72,7 @@ is_leapyear(npy_int64 year) /* * Calculates the days offset from the 1970 epoch. */ -static npy_int64 +NPY_NO_EXPORT npy_int64 get_datetimestruct_days(const npy_datetimestruct *dts) { int i, month; @@ -115,7 +117,7 @@ get_datetimestruct_days(const npy_datetimestruct *dts) days += year / 400; } - month_lengths = days_in_month[is_leapyear(dts->year)]; + month_lengths = _days_per_month_table[is_leapyear(dts->year)]; month = dts->month - 1; /* Add the months */ @@ -129,6 +131,19 @@ get_datetimestruct_days(const npy_datetimestruct *dts) return days; } +/* + * Calculates the minutes offset from the 1970 epoch. + */ +NPY_NO_EXPORT npy_int64 +get_datetimestruct_minutes(const npy_datetimestruct *dts) +{ + npy_int64 days = get_datetimestruct_days(dts) * 24 * 60; + days += dts->hour * 60; + days += dts->min; + + return days; +} + /* * Modifies '*days_' to be the day offset within the year, * and returns the year. @@ -180,7 +195,7 @@ days_to_month_number(npy_datetime days) int *month_lengths, i; year = days_to_yearsdays(&days); - month_lengths = days_in_month[is_leapyear(year)]; + month_lengths = _days_per_month_table[is_leapyear(year)]; for (i = 0; i < 12; ++i) { if (days < month_lengths[i]) { @@ -205,7 +220,7 @@ set_datetimestruct_days(npy_int64 days, npy_datetimestruct *dts) int *month_lengths, i; dts->year = days_to_yearsdays(&days); - month_lengths = days_in_month[is_leapyear(dts->year)]; + month_lengths = _days_per_month_table[is_leapyear(dts->year)]; for (i = 0; i < 12; ++i) { if (days < month_lengths[i]) { @@ -249,13 +264,6 @@ convert_datetimestruct_to_datetime(PyArray_DatetimeMetaData *meta, return -1; } - if (dts->event < 0 || dts->event >= meta->events) { - PyErr_Format(PyExc_ValueError, - "NumPy datetime event %d is outside range [0,%d)", - (int)dts->event, (int)meta->events); - return -1; - } - if (base == NPY_FR_Y) { /* Truncate to the year */ ret = dts->year - 1970; @@ -364,12 +372,6 @@ convert_datetimestruct_to_datetime(PyArray_DatetimeMetaData *meta, } } - /* Add in the event number if needed */ - if (meta->events > 1) { - /* Multiply by the number of events and put in the event number */ - ret = ret * meta->events + dts->event; - } - *out = ret; return 0; @@ -383,27 +385,12 @@ convert_datetimestruct_to_datetime(PyArray_DatetimeMetaData *meta, NPY_NO_EXPORT npy_datetime PyArray_DatetimeStructToDatetime(NPY_DATETIMEUNIT fr, npy_datetimestruct *d) { - npy_datetime ret; - PyArray_DatetimeMetaData meta; - - /* Set up a dummy metadata for the conversion */ - meta.base = fr; - meta.num = 1; - meta.events = 1; - - if (convert_datetimestruct_to_datetime(&meta, d, &ret) < 0) { - /* The caller then needs to check PyErr_Occurred() */ - return -1; - } - - return ret; + PyErr_SetString(PyExc_RuntimeError, + "The NumPy PyArray_DatetimeStructToDatetime function has " + "been removed"); + return -1; } -/* Uses Average values when frequency is Y, M, or B */ - -#define _DAYS_PER_MONTH 30.436875 -#define _DAYS_PER_YEAR 365.2425 - /*NUMPY_API * Create a timdelta value from a filled timedelta struct and resolution unit. * @@ -412,83 +399,10 @@ PyArray_DatetimeStructToDatetime(NPY_DATETIMEUNIT fr, npy_datetimestruct *d) NPY_NO_EXPORT npy_datetime PyArray_TimedeltaStructToTimedelta(NPY_DATETIMEUNIT fr, npy_timedeltastruct *d) { - npy_datetime ret; - - if (fr == NPY_FR_Y) { - ret = d->day / _DAYS_PER_YEAR; - } - else if (fr == NPY_FR_M) { - ret = d->day / _DAYS_PER_MONTH; - } - else if (fr == NPY_FR_W) { - /* This is just 7-days for now. */ - if (d->day >= 0) { - ret = d->day / 7; - } - else { - ret = (d->day - 6) / 7; - } - } - else if (fr == NPY_FR_D) { - ret = d->day; - } - else if (fr == NPY_FR_h) { - ret = d->day + d->sec / 3600; - } - else if (fr == NPY_FR_m) { - ret = d->day * (npy_int64)(1440) + d->sec / 60; - } - else if (fr == NPY_FR_s) { - ret = d->day * (npy_int64)(86400) + d->sec; - } - else if (fr == NPY_FR_ms) { - ret = d->day * (npy_int64)(86400000) + d->sec * 1000 + d->us / 1000; - } - else if (fr == NPY_FR_us) { - npy_int64 num = 86400000; - num *= (npy_int64)(1000); - ret = d->day * num + d->sec * (npy_int64)(1000000) + d->us; - } - else if (fr == NPY_FR_ns) { - npy_int64 num = 86400000; - num *= (npy_int64)(1000000); - ret = d->day * num + d->sec * (npy_int64)(1000000000) - + d->us * (npy_int64)(1000) + (d->ps / 1000); - } - else if (fr == NPY_FR_ps) { - npy_int64 num2, num1; - - num2 = 1000000; - num2 *= (npy_int64)(1000000); - num1 = (npy_int64)(86400) * num2; - - ret = d->day * num1 + d->sec * num2 + d->us * (npy_int64)(1000000) - + d->ps; - } - else if (fr == NPY_FR_fs) { - /* only 2.6 hours */ - npy_int64 num2 = 1000000000; - num2 *= (npy_int64)(1000000); - ret = d->sec * num2 + d->us * (npy_int64)(1000000000) - + d->ps * (npy_int64)(1000) + (d->as / 1000); - } - else if (fr == NPY_FR_as) { - /* only 9.2 secs */ - npy_int64 num1, num2; - - num1 = 1000000; - num1 *= (npy_int64)(1000000); - num2 = num1 * (npy_int64)(1000000); - ret = d->sec * num2 + d->us * num1 + d->ps * (npy_int64)(1000000) - + d->as; - } - else { - /* Shouldn't get here */ - PyErr_SetString(PyExc_ValueError, "invalid internal frequency"); - ret = -1; - } - - return ret; + PyErr_SetString(PyExc_RuntimeError, + "The NumPy PyArray_TimedeltaStructToTimedelta function has " + "been removed"); + return -1; } /* @@ -521,16 +435,6 @@ convert_datetime_to_datetimestruct(PyArray_DatetimeMetaData *meta, return -1; } - /* Extract the event number */ - if (meta->events > 1) { - out->event = dt % meta->events; - dt = dt / meta->events; - if (out->event < 0) { - out->event += meta->events; - --dt; - } - } - /* TODO: Change to a mechanism that avoids the potential overflow */ dt *= meta->num; @@ -751,19 +655,10 @@ NPY_NO_EXPORT void PyArray_DatetimeToDatetimeStruct(npy_datetime val, NPY_DATETIMEUNIT fr, npy_datetimestruct *result) { - PyArray_DatetimeMetaData meta; - - /* Set up a dummy metadata for the conversion */ - meta.base = fr; - meta.num = 1; - meta.events = 1; - - if (convert_datetime_to_datetimestruct(&meta, val, result) < 0) { - /* The caller needs to check PyErr_Occurred() */ - return; - } - - return; + PyErr_SetString(PyExc_RuntimeError, + "The NumPy PyArray_DatetimeToDatetimeStruct function has " + "been removed"); + memset(result, -1, sizeof(npy_datetimestruct)); } /* @@ -781,131 +676,10 @@ NPY_NO_EXPORT void PyArray_TimedeltaToTimedeltaStruct(npy_timedelta val, NPY_DATETIMEUNIT fr, npy_timedeltastruct *result) { - npy_longlong day=0; - int sec=0, us=0, ps=0, as=0; - npy_bool negative=0; - - /* - * Note that what looks like val / N and val % N for positive - * numbers maps to [val - (N-1)] / N and [N-1 + (val+1) % N] - * for negative numbers (with the 2nd value, the remainder, - * being positive in both cases). - */ - - if (val < 0) { - val = -val; - negative = 1; - } - if (fr == NPY_FR_Y) { - day = val * _DAYS_PER_YEAR; - } - else if (fr == NPY_FR_M) { - day = val * _DAYS_PER_MONTH; - } - else if (fr == NPY_FR_W) { - day = val * 7; - } - else if (fr == NPY_FR_D) { - day = val; - } - else if (fr == NPY_FR_h) { - day = val / 24; - sec = (val % 24)*3600; - } - else if (fr == NPY_FR_m) { - day = val / 1440; - sec = (val % 1440)*60; - } - else if (fr == NPY_FR_s) { - day = val / (86400); - sec = val % 86400; - } - else if (fr == NPY_FR_ms) { - day = val / 86400000; - val = val % 86400000; - sec = val / 1000; - us = (val % 1000)*1000; - } - else if (fr == NPY_FR_us) { - npy_int64 num1; - num1 = 86400000; - num1 *= 1000; - day = val / num1; - us = val % num1; - sec = us / 1000000; - us = us % 1000000; - } - else if (fr == NPY_FR_ns) { - npy_int64 num1; - num1 = 86400000; - num1 *= 1000000; - day = val / num1; - val = val % num1; - sec = val / 1000000000; - val = val % 1000000000; - us = val / 1000; - ps = (val % 1000) * (npy_int64)(1000); - } - else if (fr == NPY_FR_ps) { - npy_int64 num1, num2; - num2 = 1000000000; - num2 *= (npy_int64)(1000); - num1 = (npy_int64)(86400) * num2; - - day = val / num1; - ps = val % num1; - sec = ps / num2; - ps = ps % num2; - us = ps / 1000000; - ps = ps % 1000000; - } - else if (fr == NPY_FR_fs) { - /* entire range is only += 9.2 hours */ - npy_int64 num1, num2; - num1 = 1000000000; - num2 = num1 * (npy_int64)(1000000); - - day = 0; - sec = val / num2; - val = val % num2; - us = val / num1; - val = val % num1; - ps = val / 1000; - as = (val % 1000) * (npy_int64)(1000); - } - else if (fr == NPY_FR_as) { - /* entire range is only += 2.6 seconds */ - npy_int64 num1, num2, num3; - num1 = 1000000; - num2 = num1 * (npy_int64)(1000000); - num3 = num2 * (npy_int64)(1000000); - day = 0; - sec = val / num3; - as = val % num3; - us = as / num2; - as = as % num2; - ps = as / num1; - as = as % num1; - } - else { - PyErr_SetString(PyExc_RuntimeError, "invalid internal time resolution"); - } - - if (negative) { - result->day = -day; - result->sec = -sec; - result->us = -us; - result->ps = -ps; - result->as = -as; - } - else { - result->day = day; - result->sec = sec; - result->us = us; - result->ps = ps; - result->as = as; - } - return; + PyErr_SetString(PyExc_RuntimeError, + "The NumPy PyArray_TimedeltaToTimedeltaStruct function has " + "been removed"); + memset(result, -1, sizeof(npy_timedeltastruct)); } /* @@ -983,7 +757,6 @@ create_datetime_dtype_with_unit(int type_num, NPY_DATETIMEUNIT unit) PyArray_DatetimeMetaData meta; meta.base = unit; meta.num = 1; - meta.events = 1; return create_datetime_dtype(type_num, &meta); } @@ -1133,7 +906,6 @@ parse_datetime_metadata_from_metastr(char *metastr, Py_ssize_t len, if (len == 0) { out_meta->base = NPY_FR_GENERIC; out_meta->num = 1; - out_meta->events = 1; return 0; } @@ -1144,10 +916,10 @@ parse_datetime_metadata_from_metastr(char *metastr, Py_ssize_t len, } substrend = substr; - while (*substrend != '\0' && *substrend != ']') { + while (substrend - metastr < len && *substrend != ']') { ++substrend; } - if (*substrend == '\0' || substr == substrend) { + if (substrend - metastr == len || substr == substrend) { substr = substrend; goto bad_input; } @@ -1160,21 +932,9 @@ parse_datetime_metadata_from_metastr(char *metastr, Py_ssize_t len, substr = substrend+1; - /* Finally comes an optional number of events */ - if (substr[0] == '/' && substr[1] == '/') { - substr += 2; - - out_meta->events = (int)strtol(substr, &substrend, 10); - if (substr == substrend || *substrend != '\0') { - goto bad_input; - } - } - else if (*substr != '\0') { + if (substr - metastr != len) { goto bad_input; } - else { - out_meta->events = 1; - } return 0; @@ -1417,11 +1177,7 @@ _uint64_euclidean_gcd(npy_uint64 x, npy_uint64 y) /* * Computes the conversion factor to convert data with 'src_meta' metadata - * into data with 'dst_meta' metadata, not taking into account the events. - * - * To convert a npy_datetime or npy_timedelta, first the event number needs - * to be divided away, then it needs to be scaled by num/denom, and - * finally the event number can be added back in. + * into data with 'dst_meta' metadata. * * If overflow occurs, both out_num and out_denom are set to 0, but * no error is set. @@ -1537,58 +1293,33 @@ get_datetime_conversion_factor(PyArray_DatetimeMetaData *src_meta, */ NPY_NO_EXPORT npy_bool datetime_metadata_divides( - PyArray_Descr *dividend, - PyArray_Descr *divisor, + PyArray_DatetimeMetaData *dividend, + PyArray_DatetimeMetaData *divisor, int strict_with_nonlinear_units) { - PyArray_DatetimeMetaData *meta1, *meta2; npy_uint64 num1, num2; - /* Must be datetime types */ - if ((dividend->type_num != NPY_DATETIME && - dividend->type_num != NPY_TIMEDELTA) || - (divisor->type_num != NPY_DATETIME && - divisor->type_num != NPY_TIMEDELTA)) { - return 0; - } - - meta1 = get_datetime_metadata_from_dtype(dividend); - if (meta1 == NULL) { - PyErr_Clear(); - return 0; - } - meta2 = get_datetime_metadata_from_dtype(divisor); - if (meta2 == NULL) { - PyErr_Clear(); - return 0; - } - /* Generic units divide into anything */ - if (meta2->base == NPY_FR_GENERIC) { + if (divisor->base == NPY_FR_GENERIC) { return 1; } /* Non-generic units never divide into generic units */ - else if (meta1->base == NPY_FR_GENERIC) { - return 0; - } - - /* Events must match */ - if (meta1->events != meta2->events) { + else if (dividend->base == NPY_FR_GENERIC) { return 0; } - num1 = (npy_uint64)meta1->num; - num2 = (npy_uint64)meta2->num; + num1 = (npy_uint64)dividend->num; + num2 = (npy_uint64)divisor->num; /* If the bases are different, factor in a conversion */ - if (meta1->base != meta2->base) { + if (dividend->base != divisor->base) { /* * Years and Months are incompatible with * all other units (except years and months are compatible * with each other). */ - if (meta1->base == NPY_FR_Y) { - if (meta2->base == NPY_FR_M) { + if (dividend->base == NPY_FR_Y) { + if (divisor->base == NPY_FR_M) { num1 *= 12; } else if (strict_with_nonlinear_units) { @@ -1599,8 +1330,8 @@ datetime_metadata_divides( return 1; } } - else if (meta2->base == NPY_FR_Y) { - if (meta1->base == NPY_FR_M) { + else if (divisor->base == NPY_FR_Y) { + if (dividend->base == NPY_FR_M) { num2 *= 12; } else if (strict_with_nonlinear_units) { @@ -1611,7 +1342,7 @@ datetime_metadata_divides( return 1; } } - else if (meta1->base == NPY_FR_M || meta2->base == NPY_FR_M) { + else if (dividend->base == NPY_FR_M || divisor->base == NPY_FR_M) { if (strict_with_nonlinear_units) { return 0; } @@ -1622,14 +1353,14 @@ datetime_metadata_divides( } /* Take the greater base (unit sizes are decreasing in enum) */ - if (meta1->base > meta2->base) { - num2 *= get_datetime_units_factor(meta2->base, meta1->base); + if (dividend->base > divisor->base) { + num2 *= get_datetime_units_factor(divisor->base, dividend->base); if (num2 == 0) { return 0; } } else { - num1 *= get_datetime_units_factor(meta1->base, meta2->base); + num1 *= get_datetime_units_factor(dividend->base, divisor->base); if (num1 == 0) { return 0; } @@ -1644,6 +1375,222 @@ datetime_metadata_divides( return (num1 % num2) == 0; } +/* + * This provides the casting rules for the DATETIME data type units. + * + * Notably, there is a barrier between 'date units' and 'time units' + * for all but 'unsafe' casting. + */ +NPY_NO_EXPORT npy_bool +can_cast_datetime64_units(NPY_DATETIMEUNIT src_unit, + NPY_DATETIMEUNIT dst_unit, + NPY_CASTING casting) +{ + switch (casting) { + /* Allow anything with unsafe casting */ + case NPY_UNSAFE_CASTING: + return 1; + + /* + * Only enforce the 'date units' vs 'time units' barrier with + * 'same_kind' casting. + */ + case NPY_SAME_KIND_CASTING: + if (src_unit == NPY_FR_GENERIC || dst_unit == NPY_FR_GENERIC) { + return src_unit == dst_unit; + } + else { + return (src_unit <= NPY_FR_D && dst_unit <= NPY_FR_D) || + (src_unit > NPY_FR_D && dst_unit > NPY_FR_D); + } + + /* + * Enforce the 'date units' vs 'time units' barrier and that + * casting is only allowed towards more precise units with + * 'safe' casting. + */ + case NPY_SAFE_CASTING: + if (src_unit == NPY_FR_GENERIC || dst_unit == NPY_FR_GENERIC) { + return src_unit == dst_unit; + } + else { + return (src_unit <= dst_unit) && + ((src_unit <= NPY_FR_D && dst_unit <= NPY_FR_D) || + (src_unit > NPY_FR_D && dst_unit > NPY_FR_D)); + } + + /* Enforce equality with 'no' or 'equiv' casting */ + default: + return src_unit == dst_unit; + } +} + +/* + * This provides the casting rules for the TIMEDELTA data type units. + * + * Notably, there is a barrier between the nonlinear years and + * months units, and all the other units. + */ +NPY_NO_EXPORT npy_bool +can_cast_timedelta64_units(NPY_DATETIMEUNIT src_unit, + NPY_DATETIMEUNIT dst_unit, + NPY_CASTING casting) +{ + switch (casting) { + /* Allow anything with unsafe casting */ + case NPY_UNSAFE_CASTING: + return 1; + + /* + * Only enforce the 'date units' vs 'time units' barrier with + * 'same_kind' casting. + */ + case NPY_SAME_KIND_CASTING: + if (src_unit == NPY_FR_GENERIC || dst_unit == NPY_FR_GENERIC) { + return src_unit == dst_unit; + } + else { + return (src_unit <= NPY_FR_M && dst_unit <= NPY_FR_M) || + (src_unit > NPY_FR_M && dst_unit > NPY_FR_M); + } + + /* + * Enforce the 'date units' vs 'time units' barrier and that + * casting is only allowed towards more precise units with + * 'safe' casting. + */ + case NPY_SAFE_CASTING: + if (src_unit == NPY_FR_GENERIC || dst_unit == NPY_FR_GENERIC) { + return src_unit == dst_unit; + } + else { + return (src_unit <= dst_unit) && + ((src_unit <= NPY_FR_M && dst_unit <= NPY_FR_M) || + (src_unit > NPY_FR_M && dst_unit > NPY_FR_M)); + } + + /* Enforce equality with 'no' or 'equiv' casting */ + default: + return src_unit == dst_unit; + } +} + +/* + * This provides the casting rules for the DATETIME data type metadata. + */ +NPY_NO_EXPORT npy_bool +can_cast_datetime64_metadata(PyArray_DatetimeMetaData *src_meta, + PyArray_DatetimeMetaData *dst_meta, + NPY_CASTING casting) +{ + switch (casting) { + case NPY_UNSAFE_CASTING: + return 1; + + case NPY_SAME_KIND_CASTING: + return can_cast_datetime64_units(src_meta->base, dst_meta->base, + casting); + + case NPY_SAFE_CASTING: + return can_cast_datetime64_units(src_meta->base, dst_meta->base, + casting) && + datetime_metadata_divides(src_meta, dst_meta, 0); + + default: + return src_meta->base == dst_meta->base && + src_meta->num == dst_meta->num; + } +} + +/* + * This provides the casting rules for the TIMEDELTA data type metadata. + */ +NPY_NO_EXPORT npy_bool +can_cast_timedelta64_metadata(PyArray_DatetimeMetaData *src_meta, + PyArray_DatetimeMetaData *dst_meta, + NPY_CASTING casting) +{ + switch (casting) { + case NPY_UNSAFE_CASTING: + return 1; + + case NPY_SAME_KIND_CASTING: + return can_cast_timedelta64_units(src_meta->base, dst_meta->base, + casting); + + case NPY_SAFE_CASTING: + return can_cast_timedelta64_units(src_meta->base, dst_meta->base, + casting) && + datetime_metadata_divides(src_meta, dst_meta, 1); + + default: + return src_meta->base == dst_meta->base && + src_meta->num == dst_meta->num; + } +} + +/* + * Tests whether a datetime64 can be cast from the source metadata + * to the destination metadata according to the specified casting rule. + * + * Returns -1 if an exception was raised, 0 otherwise. + */ +NPY_NO_EXPORT int +raise_if_datetime64_metadata_cast_error(char *object_type, + PyArray_DatetimeMetaData *src_meta, + PyArray_DatetimeMetaData *dst_meta, + NPY_CASTING casting) +{ + if (can_cast_datetime64_metadata(src_meta, dst_meta, casting)) { + return 0; + } + else { + PyObject *errmsg; + errmsg = PyUString_FromFormat("Cannot cast %s " + "from metadata ", object_type); + errmsg = append_metastr_to_string(src_meta, 0, errmsg); + PyUString_ConcatAndDel(&errmsg, + PyUString_FromString(" to ")); + errmsg = append_metastr_to_string(dst_meta, 0, errmsg); + PyUString_ConcatAndDel(&errmsg, + PyUString_FromFormat(" according to the rule %s", + npy_casting_to_string(casting))); + PyErr_SetObject(PyExc_TypeError, errmsg); + return -1; + } +} + +/* + * Tests whether a timedelta64 can be cast from the source metadata + * to the destination metadata according to the specified casting rule. + * + * Returns -1 if an exception was raised, 0 otherwise. + */ +NPY_NO_EXPORT int +raise_if_timedelta64_metadata_cast_error(char *object_type, + PyArray_DatetimeMetaData *src_meta, + PyArray_DatetimeMetaData *dst_meta, + NPY_CASTING casting) +{ + if (can_cast_timedelta64_metadata(src_meta, dst_meta, casting)) { + return 0; + } + else { + PyObject *errmsg; + errmsg = PyUString_FromFormat("Cannot cast %s " + "from metadata ", object_type); + errmsg = append_metastr_to_string(src_meta, 0, errmsg); + PyUString_ConcatAndDel(&errmsg, + PyUString_FromString(" to ")); + errmsg = append_metastr_to_string(dst_meta, 0, errmsg); + PyUString_ConcatAndDel(&errmsg, + PyUString_FromFormat(" according to the rule %s", + npy_casting_to_string(casting))); + PyErr_SetObject(PyExc_TypeError, errmsg); + return -1; + } +} + /* * Computes the GCD of the two date-time metadata values. Raises * an exception if there is no reasonable GCD, such as with @@ -1663,7 +1610,6 @@ compute_datetime_metadata_greatest_common_divisor( { NPY_DATETIMEUNIT base; npy_uint64 num1, num2, num; - int events = 1; /* If either unit is generic, adopt the metadata from the other one */ if (meta1->base == NPY_FR_GENERIC) { @@ -1675,14 +1621,6 @@ compute_datetime_metadata_greatest_common_divisor( return 0; } - /* Take the maximum of the events */ - if (meta1->events > meta2->events) { - events = meta1->events; - } - else { - events = meta2->events; - } - num1 = (npy_uint64)meta1->num; num2 = (npy_uint64)meta2->num; @@ -1767,7 +1705,6 @@ compute_datetime_metadata_greatest_common_divisor( if (out_meta->num <= 0 || num != (npy_uint64)out_meta->num) { goto units_overflow; } - out_meta->events = events; return 0; @@ -1984,7 +1921,7 @@ convert_datetime_metadata_to_tuple(PyArray_DatetimeMetaData *meta) { PyObject *dt_tuple; - dt_tuple = PyTuple_New(3); + dt_tuple = PyTuple_New(2); if (dt_tuple == NULL) { return NULL; } @@ -1993,8 +1930,6 @@ convert_datetime_metadata_to_tuple(PyArray_DatetimeMetaData *meta) PyBytes_FromString(_datetime_strings[meta->base])); PyTuple_SET_ITEM(dt_tuple, 1, PyInt_FromLong(meta->num)); - PyTuple_SET_ITEM(dt_tuple, 2, - PyInt_FromLong(meta->events)); return dt_tuple; } @@ -2021,9 +1956,9 @@ convert_datetime_metadata_tuple_to_datetime_metadata(PyObject *tuple, } tuple_size = PyTuple_GET_SIZE(tuple); - if (tuple_size < 3 || tuple_size > 4) { + if (tuple_size < 2 || tuple_size > 4) { PyErr_SetString(PyExc_TypeError, - "Require tuple of size 3 or 4 for " + "Require tuple of size 2 to 4 for " "tuple to NumPy datetime metadata conversion"); return -1; } @@ -2044,24 +1979,14 @@ convert_datetime_metadata_tuple_to_datetime_metadata(PyObject *tuple, return -1; } - if (tuple_size == 3) { - out_meta->events = PyInt_AsLong(PyTuple_GET_ITEM(tuple, 2)); - if (out_meta->events == -1 && PyErr_Occurred()) { - return -1; - } - } - else { + if (tuple_size == 4) { den = PyInt_AsLong(PyTuple_GET_ITEM(tuple, 2)); if (den == -1 && PyErr_Occurred()) { return -1; } - out_meta->events = PyInt_AsLong(PyTuple_GET_ITEM(tuple, 3)); - if (out_meta->events == -1 && PyErr_Occurred()) { - return -1; - } } - if (out_meta->num <= 0 || out_meta->events <= 0 || den <= 0) { + if (out_meta->num <= 0 || den <= 0) { PyErr_SetString(PyExc_TypeError, "Invalid tuple values for " "tuple to NumPy datetime metadata conversion"); @@ -2146,9 +2071,6 @@ convert_pyobject_to_datetime_metadata(PyObject *obj, return -1; } - /* extended_unit is only 'num' and 'base', we have to fill the rest */ - out_meta->events = 1; - return 0; } @@ -2158,7 +2080,7 @@ convert_pyobject_to_datetime_metadata(PyObject *obj, * 'ret' is a PyUString containing the datetime string, and this * function appends the metadata string to it. * - * If 'skip_brackets' is true, skips the '[]' when events == 1. + * If 'skip_brackets' is true, skips the '[]'. * * This function steals the reference 'ret' */ @@ -2168,7 +2090,7 @@ append_metastr_to_string(PyArray_DatetimeMetaData *meta, PyObject *ret) { PyObject *res; - int num, events; + int num; char *basestr; if (ret == NULL) { @@ -2188,7 +2110,6 @@ append_metastr_to_string(PyArray_DatetimeMetaData *meta, } num = meta->num; - events = meta->events; if (meta->base >= 0 && meta->base < NPY_DATETIME_NUMUNITS) { basestr = _datetime_strings[meta->base]; } @@ -2199,7 +2120,7 @@ append_metastr_to_string(PyArray_DatetimeMetaData *meta, } if (num == 1) { - if (skip_brackets && events == 1) { + if (skip_brackets) { res = PyUString_FromFormat("%s", basestr); } else { @@ -2207,7 +2128,7 @@ append_metastr_to_string(PyArray_DatetimeMetaData *meta, } } else { - if (skip_brackets && events == 1) { + if (skip_brackets) { res = PyUString_FromFormat("%d%s", num, basestr); } else { @@ -2215,10 +2136,6 @@ append_metastr_to_string(PyArray_DatetimeMetaData *meta, } } - if (events != 1) { - PyUString_ConcatAndDel(&res, - PyUString_FromFormat("//%d", events)); - } PyUString_ConcatAndDel(&ret, res); return ret; } @@ -2287,12 +2204,12 @@ add_minutes_to_datetimestruct(npy_datetimestruct *dts, int minutes) dts->month = 12; } isleap = is_leapyear(dts->year); - dts->day += days_in_month[isleap][dts->month-1]; + dts->day += _days_per_month_table[isleap][dts->month-1]; } else if (dts->day > 28) { isleap = is_leapyear(dts->year); - if (dts->day > days_in_month[isleap][dts->month-1]) { - dts->day -= days_in_month[isleap][dts->month-1]; + if (dts->day > _days_per_month_table[isleap][dts->month-1]) { + dts->day -= _days_per_month_table[isleap][dts->month-1]; dts->month++; if (dts->month > 12) { dts->year++; @@ -2303,1204 +2220,30 @@ add_minutes_to_datetimestruct(npy_datetimestruct *dts, int minutes) } /* - * Parses (almost) standard ISO 8601 date strings. The differences are: - * - * + After the date and time, may place a ' ' followed by an event number. - * + The date "20100312" is parsed as the year 20100312, not as - * equivalent to "2010-03-12". The '-' in the dates are not optional. - * + Only seconds may have a decimal point, with up to 18 digits after it - * (maximum attoseconds precision). - * + Either a 'T' as in ISO 8601 or a ' ' may be used to separate - * the date and the time. Both are treated equivalently. - * + Doesn't (yet) handle the "YYYY-DDD" or "YYYY-Www" formats. - * + Doesn't handle leap seconds (seconds value has 60 in these cases). - * + Doesn't handle 24:00:00 as synonym for midnight (00:00:00) tomorrow - * + Accepts special values "NaT" (not a time), "Today", (current - * day according to local time) and "Now" (current time in UTC). + * Tests for and converts a Python datetime.datetime or datetime.date + * object into a NumPy npy_datetimestruct. * - * 'str' must be a NULL-terminated string, and 'len' must be its length. - * 'unit' should contain -1 if the unit is unknown, or the unit - * which will be used if it is. + * While the C API has PyDate_* and PyDateTime_* functions, the following + * implementation just asks for attributes, and thus supports + * datetime duck typing. The tzinfo time zone conversion would require + * this style of access anyway. * - * 'out' gets filled with the parsed date-time. - * 'out_local' gets set to 1 if the parsed time was in local time, - * to 0 otherwise. The values 'now' and 'today' don't get counted - * as local, and neither do UTC +/-#### timezone offsets, because - * they aren't using the computer's local timezone offset. - * 'out_bestunit' gives a suggested unit based on the amount of - * resolution provided in the string, or -1 for NaT. - * 'out_special' gets set to 1 if the parsed time was 'today', - * 'now', or ''/'NaT'. For 'today', the unit recommended is - * 'D', for 'now', the unit recommended is 's', and for 'NaT' - * the unit recommended is 'Y'. + * 'out_bestunit' gives a suggested unit based on whether the object + * was a datetime.date or datetime.datetime object. * + * If 'apply_tzinfo' is 1, this function uses the tzinfo to convert + * to UTC time, otherwise it returns the struct with the local time. * - * Returns 0 on success, -1 on failure. + * Returns -1 on error, 0 on success, and 1 (with no error set) + * if obj doesn't have the neeeded date or datetime attributes. */ NPY_NO_EXPORT int -parse_iso_8601_date(char *str, int len, - NPY_DATETIMEUNIT unit, - npy_datetimestruct *out, - npy_bool *out_local, - NPY_DATETIMEUNIT *out_bestunit, - npy_bool *out_special) +convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, + NPY_DATETIMEUNIT *out_bestunit, + int apply_tzinfo) { - int year_leap = 0; - int i, numdigits; - char *substr, sublen; - - /* Initialize the output to all zeros */ - memset(out, 0, sizeof(npy_datetimestruct)); - out->month = 1; - out->day = 1; - - /* The empty string and case-variants of "NaT" parse to not-a-time */ - if (len <= 0 || (len == 3 && - tolower(str[0]) == 'n' && - tolower(str[1]) == 'a' && - tolower(str[2]) == 't')) { - out->year = NPY_DATETIME_NAT; - - /* - * Indicate that this was a special value, and - * recommend generic units. - */ - if (out_local != NULL) { - *out_local = 0; - } - if (out_bestunit != NULL) { - *out_bestunit = NPY_FR_GENERIC; - } - if (out_special != NULL) { - *out_special = 1; - } - - return 0; - } - - if (unit == NPY_FR_GENERIC) { - PyErr_SetString(PyExc_ValueError, - "Cannot create a NumPy datetime other than NaT " - "with generic units"); - return -1; - } - - /* - * The string "today" resolves to midnight of today's local date in UTC. - * This is perhaps a little weird, but done so that further truncation - * to a 'datetime64[D]' type produces the date you expect, rather than - * switching to an adjacent day depending on the current time and your - * timezone. - */ - if (len == 5 && tolower(str[0]) == 't' && - tolower(str[1]) == 'o' && - tolower(str[2]) == 'd' && - tolower(str[3]) == 'a' && - tolower(str[4]) == 'y') { - time_t rawtime = 0; - struct tm tm_; - - /* 'today' only works for units of days or larger */ - if (unit != -1 && unit > NPY_FR_D) { - PyErr_SetString(PyExc_ValueError, - "Special value 'today' can only be converted " - "to a NumPy datetime with 'D' or larger units"); - return -1; - } - - time(&rawtime); -#if defined(_WIN32) - if (localtime_s(&tm_, &rawtime) != 0) { - PyErr_SetString(PyExc_OSError, "Failed to use localtime_s to " - "get local time"); - return -1; - } -#else - /* Other platforms may require something else */ - if (localtime_r(&rawtime, &tm_) == NULL) { - PyErr_SetString(PyExc_OSError, "Failed to use localtime_r to " - "get local time"); - return -1; - } -#endif - out->year = tm_.tm_year + 1900; - out->month = tm_.tm_mon + 1; - out->day = tm_.tm_mday; - - /* - * Indicate that this was a special value, and - * is a date (unit 'D'). - */ - if (out_local != NULL) { - *out_local = 0; - } - if (out_bestunit != NULL) { - *out_bestunit = NPY_FR_D; - } - if (out_special != NULL) { - *out_special = 1; - } - - return 0; - } - - /* The string "now" resolves to the current UTC time */ - if (len == 3 && tolower(str[0]) == 'n' && - tolower(str[1]) == 'o' && - tolower(str[2]) == 'w') { - time_t rawtime = 0; - PyArray_DatetimeMetaData meta; - - /* 'now' only works for units of hours or smaller */ - if (unit != -1 && unit < NPY_FR_h) { - PyErr_SetString(PyExc_ValueError, - "Special value 'now' can only be converted " - "to a NumPy datetime with 'h' or smaller units"); - return -1; - } - - time(&rawtime); - - /* Set up a dummy metadata for the conversion */ - meta.base = NPY_FR_s; - meta.num = 1; - meta.events = 1; - - /* - * Indicate that this was a special value, and - * use 's' because the time() function has resolution - * seconds. - */ - if (out_local != NULL) { - *out_local = 0; - } - if (out_bestunit != NULL) { - *out_bestunit = NPY_FR_s; - } - if (out_special != NULL) { - *out_special = 1; - } - - return convert_datetime_to_datetimestruct(&meta, rawtime, out); - } - - /* Anything else isn't a special value */ - if (out_special != NULL) { - *out_special = 0; - } - - substr = str; - sublen = len; - - /* Skip leading whitespace */ - while (sublen > 0 && isspace(*substr)) { - ++substr; - --sublen; - } - - /* Leading '-' sign for negative year */ - if (*substr == '-') { - ++substr; - --sublen; - } - - if (sublen == 0) { - goto parse_error; - } - - /* PARSE THE YEAR (digits until the '-' character) */ - out->year = 0; - while (sublen > 0 && isdigit(*substr)) { - out->year = 10 * out->year + (*substr - '0'); - ++substr; - --sublen; - } - - /* Negate the year if necessary */ - if (str[0] == '-') { - out->year = -out->year; - } - /* Check whether it's a leap-year */ - year_leap = is_leapyear(out->year); - - /* Next character must be a '-' or the end of the string */ - if (sublen == 0) { - if (out_local != NULL) { - *out_local = 0; - } - if (out_bestunit != NULL) { - *out_bestunit = NPY_FR_Y; - } - goto finish; - } - else if (*substr == '-') { - ++substr; - --sublen; - } - else { - goto parse_error; - } - - /* Can't have a trailing '-' */ - if (sublen == 0) { - goto parse_error; - } - - /* PARSE THE MONTH (2 digits) */ - if (sublen >= 2 && isdigit(substr[0]) && isdigit(substr[1])) { - out->month = 10 * (substr[0] - '0') + (substr[1] - '0'); - - if (out->month < 1 || out->month > 12) { - PyErr_Format(PyExc_ValueError, - "Month out of range in datetime string \"%s\"", str); - goto error; - } - substr += 2; - sublen -= 2; - } - else { - goto parse_error; - } - - /* Next character must be a '-' or the end of the string */ - if (sublen == 0) { - if (out_local != NULL) { - *out_local = 0; - } - if (out_bestunit != NULL) { - *out_bestunit = NPY_FR_M; - } - goto finish; - } - else if (*substr == '-') { - ++substr; - --sublen; - } - else { - goto parse_error; - } - - /* Can't have a trailing '-' */ - if (sublen == 0) { - goto parse_error; - } - - /* PARSE THE DAY (2 digits) */ - if (sublen >= 2 && isdigit(substr[0]) && isdigit(substr[1])) { - out->day = 10 * (substr[0] - '0') + (substr[1] - '0'); - - if (out->day < 1 || - out->day > days_in_month[year_leap][out->month-1]) { - PyErr_Format(PyExc_ValueError, - "Day out of range in datetime string \"%s\"", str); - goto error; - } - substr += 2; - sublen -= 2; - } - else { - goto parse_error; - } - - /* Next character must be a 'T', ' ', or end of string */ - if (sublen == 0) { - if (out_local != NULL) { - *out_local = 0; - } - if (out_bestunit != NULL) { - *out_bestunit = NPY_FR_D; - } - goto finish; - } - else if (*substr != 'T' && *substr != ' ') { - goto parse_error; - } - else { - ++substr; - --sublen; - } - - /* PARSE THE HOURS (2 digits) */ - if (sublen >= 2 && isdigit(substr[0]) && isdigit(substr[1])) { - out->hour = 10 * (substr[0] - '0') + (substr[1] - '0'); - - if (out->hour < 0 || out->hour >= 24) { - PyErr_Format(PyExc_ValueError, - "Hours out of range in datetime string \"%s\"", str); - goto error; - } - substr += 2; - sublen -= 2; - } - else { - goto parse_error; - } - - /* Next character must be a ':' or the end of the string */ - if (sublen > 0 && *substr == ':') { - ++substr; - --sublen; - } - else { - if (out_bestunit != NULL) { - *out_bestunit = NPY_FR_h; - } - goto parse_timezone; - } - - /* Can't have a trailing ':' */ - if (sublen == 0) { - goto parse_error; - } - - /* PARSE THE MINUTES (2 digits) */ - if (sublen >= 2 && isdigit(substr[0]) && isdigit(substr[1])) { - out->min = 10 * (substr[0] - '0') + (substr[1] - '0'); - - if (out->hour < 0 || out->min >= 60) { - PyErr_Format(PyExc_ValueError, - "Minutes out of range in datetime string \"%s\"", str); - goto error; - } - substr += 2; - sublen -= 2; - } - else { - goto parse_error; - } - - /* Next character must be a ':' or the end of the string */ - if (sublen > 0 && *substr == ':') { - ++substr; - --sublen; - } - else { - if (out_bestunit != NULL) { - *out_bestunit = NPY_FR_m; - } - goto parse_timezone; - } - - /* Can't have a trailing ':' */ - if (sublen == 0) { - goto parse_error; - } - - /* PARSE THE SECONDS (2 digits) */ - if (sublen >= 2 && isdigit(substr[0]) && isdigit(substr[1])) { - out->sec = 10 * (substr[0] - '0') + (substr[1] - '0'); - - if (out->sec < 0 || out->sec >= 60) { - PyErr_Format(PyExc_ValueError, - "Seconds out of range in datetime string \"%s\"", str); - goto error; - } - substr += 2; - sublen -= 2; - } - else { - goto parse_error; - } - - /* Next character may be a '.' indicating fractional seconds */ - if (sublen > 0 && *substr == '.') { - ++substr; - --sublen; - } - else { - if (out_bestunit != NULL) { - *out_bestunit = NPY_FR_s; - } - goto parse_timezone; - } - - /* PARSE THE MICROSECONDS (0 to 6 digits) */ - numdigits = 0; - for (i = 0; i < 6; ++i) { - out->us *= 10; - if (sublen > 0 && isdigit(*substr)) { - out->us += (*substr - '0'); - ++substr; - --sublen; - ++numdigits; - } - } - - if (sublen == 0 || !isdigit(*substr)) { - if (out_bestunit != NULL) { - if (numdigits > 3) { - *out_bestunit = NPY_FR_us; - } - else { - *out_bestunit = NPY_FR_ms; - } - } - goto parse_timezone; - } - - /* PARSE THE PICOSECONDS (0 to 6 digits) */ - numdigits = 0; - for (i = 0; i < 6; ++i) { - out->ps *= 10; - if (sublen > 0 && isdigit(*substr)) { - out->ps += (*substr - '0'); - ++substr; - --sublen; - ++numdigits; - } - } - - if (sublen == 0 || !isdigit(*substr)) { - if (out_bestunit != NULL) { - if (numdigits > 3) { - *out_bestunit = NPY_FR_ps; - } - else { - *out_bestunit = NPY_FR_ns; - } - } - goto parse_timezone; - } - - /* PARSE THE ATTOSECONDS (0 to 6 digits) */ - numdigits = 0; - for (i = 0; i < 6; ++i) { - out->as *= 10; - if (sublen > 0 && isdigit(*substr)) { - out->as += (*substr - '0'); - ++substr; - --sublen; - ++numdigits; - } - } - - if (out_bestunit != NULL) { - if (numdigits > 3) { - *out_bestunit = NPY_FR_as; - } - else { - *out_bestunit = NPY_FR_fs; - } - } - -parse_timezone: - if (sublen == 0) { - /* - * ISO 8601 states to treat date-times without a timezone offset - * or 'Z' for UTC as local time. The C standard libary functions - * mktime and gmtime allow us to do this conversion. - * - * Only do this timezone adjustment for recent and future years. - */ - if (out->year > 1900 && out->year < 10000) { - time_t rawtime = 0; - struct tm tm_; - - tm_.tm_sec = out->sec; - tm_.tm_min = out->min; - tm_.tm_hour = out->hour; - tm_.tm_mday = out->day; - tm_.tm_mon = out->month - 1; - tm_.tm_year = out->year - 1900; - tm_.tm_isdst = -1; - - /* mktime converts a local 'struct tm' into a time_t */ - rawtime = mktime(&tm_); - if (rawtime == -1) { - PyErr_SetString(PyExc_OSError, "Failed to use mktime to " - "convert local time to UTC"); - goto error; - } - - /* gmtime converts a 'time_t' into a UTC 'struct tm' */ -#if defined(_WIN32) - if (gmtime_s(&tm_, &rawtime) != 0) { - PyErr_SetString(PyExc_OSError, "Failed to use gmtime_s to " - "get a UTC time"); - goto error; - } -#else - /* Other platforms may require something else */ - if (gmtime_r(&rawtime, &tm_) == NULL) { - PyErr_SetString(PyExc_OSError, "Failed to use gmtime_r to " - "get a UTC time"); - goto error; - } -#endif - out->sec = tm_.tm_sec; - out->min = tm_.tm_min; - out->hour = tm_.tm_hour; - out->day = tm_.tm_mday; - out->month = tm_.tm_mon + 1; - out->year = tm_.tm_year + 1900; - } - - /* Since neither "Z" nor a time-zone was specified, it's local */ - if (out_local != NULL) { - *out_local = 1; - } - - goto finish; - } - - /* UTC specifier */ - if (*substr == 'Z') { - /* "Z" means not local */ - if (out_local != NULL) { - *out_local = 0; - } - - if (sublen == 1) { - goto finish; - } - else { - ++substr; - --sublen; - } - } - /* Time zone offset */ - else if (*substr == '-' || *substr == '+') { - int offset_neg = 0, offset_hour = 0, offset_minute = 0; - - /* - * Since "local" means local with respect to the current - * machine, we say this is non-local. - */ - if (out_local != NULL) { - *out_local = 0; - } - - if (*substr == '-') { - offset_neg = 1; - } - ++substr; - --sublen; - - /* The hours offset */ - if (sublen >= 2 && isdigit(substr[0]) && isdigit(substr[1])) { - offset_hour = 10 * (substr[0] - '0') + (substr[1] - '0'); - substr += 2; - sublen -= 2; - if (offset_hour >= 24) { - PyErr_Format(PyExc_ValueError, - "Timezone hours offset out of range " - "in datetime string \"%s\"", str); - goto error; - } - } - else { - goto parse_error; - } - - /* The minutes offset is optional */ - if (sublen > 0) { - /* Optional ':' */ - if (*substr == ':') { - ++substr; - --sublen; - } - - /* The minutes offset (at the end of the string) */ - if (sublen >= 2 && isdigit(substr[0]) && isdigit(substr[1])) { - offset_minute = 10 * (substr[0] - '0') + (substr[1] - '0'); - substr += 2; - sublen -= 2; - if (offset_minute >= 60) { - PyErr_Format(PyExc_ValueError, - "Timezone minutes offset out of range " - "in datetime string \"%s\"", str); - goto error; - } - } - else { - goto parse_error; - } - } - - /* Apply the time zone offset */ - if (offset_neg) { - offset_hour = -offset_hour; - offset_minute = -offset_minute; - } - add_minutes_to_datetimestruct(out, -60 * offset_hour - offset_minute); - } - - /* Skip trailing whitespace */ - while (sublen > 0 && isspace(*substr)) { - ++substr; - --sublen; - } - - if (sublen != 0) { - goto parse_error; - } - -finish: - return 0; - -parse_error: - PyErr_Format(PyExc_ValueError, - "Error parsing datetime string \"%s\" at position %d", - str, (int)(substr-str)); - return -1; - -error: - return -1; -} - -/* - * Provides a string length to use for converting datetime - * objects with the given local and unit settings. - */ -NPY_NO_EXPORT int -get_datetime_iso_8601_strlen(int local, NPY_DATETIMEUNIT base) -{ - int len = 0; - - /* If no unit is provided, return the maximum length */ - if (base == -1) { - return NPY_DATETIME_MAX_ISO8601_STRLEN; - } - - switch (base) { - /* Generic units can only be used to represent NaT */ - case NPY_FR_GENERIC: - return 4; - case NPY_FR_as: - len += 3; /* "###" */ - case NPY_FR_fs: - len += 3; /* "###" */ - case NPY_FR_ps: - len += 3; /* "###" */ - case NPY_FR_ns: - len += 3; /* "###" */ - case NPY_FR_us: - len += 3; /* "###" */ - case NPY_FR_ms: - len += 4; /* ".###" */ - case NPY_FR_s: - len += 3; /* ":##" */ - case NPY_FR_m: - len += 3; /* ":##" */ - case NPY_FR_h: - len += 3; /* "T##" */ - case NPY_FR_D: - case NPY_FR_W: - len += 3; /* "-##" */ - case NPY_FR_M: - len += 3; /* "-##" */ - case NPY_FR_Y: - len += 21; /* 64-bit year */ - break; - } - - if (base >= NPY_FR_h) { - if (local) { - len += 5; /* "+####" or "-####" */ - } - else { - len += 1; /* "Z" */ - } - } - - len += 1; /* NULL terminator */ - - return len; -} - -/* - * Converts an npy_datetimestruct to an (almost) ISO 8601 - * NULL-terminated string. - * - * If 'local' is non-zero, it produces a string in local time with - * a +-#### timezone offset, otherwise it uses timezone Z (UTC). - * - * 'base' restricts the output to that unit. Set 'base' to - * -1 to auto-detect a base after which all the values are zero. - * - * 'tzoffset' is used if 'local' is enabled, and 'tzoffset' is - * set to a value other than -1. This is a manual override for - * the local time zone to use, as an offset in minutes. - * - * Returns 0 on success, -1 on failure (for example if the output - * string was too short). - */ -NPY_NO_EXPORT int -make_iso_8601_date(npy_datetimestruct *dts, char *outstr, int outlen, - int local, NPY_DATETIMEUNIT base, int tzoffset) -{ - npy_datetimestruct dts_local; - int timezone_offset = 0; - - char *substr = outstr, sublen = outlen; - int tmplen; - - /* Handle NaT, and treat a datetime with generic units as NaT */ - if (dts->year == NPY_DATETIME_NAT || base == NPY_FR_GENERIC) { - if (outlen < 4) { - goto string_too_short; - } - outstr[0] = 'N'; - outstr[0] = 'a'; - outstr[0] = 'T'; - outstr[0] = '\0'; - - return 0; - } - - /* Only do local time within a reasonable year range */ - if ((dts->year <= 1900 || dts->year >= 10000) && tzoffset == -1) { - local = 0; - } - - /* Automatically detect a good unit */ - if (base == -1) { - if (dts->as % 1000 != 0) { - base = NPY_FR_as; - } - else if (dts->as != 0) { - base = NPY_FR_fs; - } - else if (dts->ps % 1000 != 0) { - base = NPY_FR_ps; - } - else if (dts->ps != 0) { - base = NPY_FR_ns; - } - else if (dts->us % 1000 != 0) { - base = NPY_FR_us; - } - else if (dts->us != 0) { - base = NPY_FR_ms; - } - else if (dts->sec != 0) { - base = NPY_FR_s; - } - /* - * hours and minutes don't get split up by default, and printing - * in local time forces minutes - */ - else if (local || dts->min != 0 || dts->hour != 0) { - base = NPY_FR_m; - } - /* dates don't get split up by default */ - else { - base = NPY_FR_D; - } - } - /* - * Print weeks with the same precision as days. - * - * TODO: Could print weeks with YYYY-Www format if the week - * epoch is a Monday. - */ - else if (base == NPY_FR_W) { - base = NPY_FR_D; - } - - /* Printed dates have no time zone */ - if (base < NPY_FR_h) { - local = 0; - } - - /* Use the C API to convert from UTC to local time */ - if (local && tzoffset == -1) { - time_t rawtime = 0, localrawtime; - struct tm tm_; - - /* - * Convert everything in 'dts' to a time_t, to minutes precision. - * This is POSIX time, which skips leap-seconds, but because - * we drop the seconds value from the npy_datetimestruct, everything - * is ok for this operation. - */ - rawtime = (time_t)get_datetimestruct_days(dts) * 24 * 60 * 60; - rawtime += dts->hour * 60 * 60; - rawtime += dts->min * 60; - - /* localtime converts a 'time_t' into a local 'struct tm' */ -#if defined(_WIN32) - if (localtime_s(&tm_, &rawtime) != 0) { - PyErr_SetString(PyExc_OSError, "Failed to use localtime_s to " - "get a local time"); - return -1; - } -#else - /* Other platforms may require something else */ - if (localtime_r(&rawtime, &tm_) == NULL) { - PyErr_SetString(PyExc_OSError, "Failed to use localtime_r to " - "get a local time"); - return -1; - } -#endif - /* Make a copy of the npy_datetimestruct we can modify */ - dts_local = *dts; - - /* Copy back all the values except seconds */ - dts_local.min = tm_.tm_min; - dts_local.hour = tm_.tm_hour; - dts_local.day = tm_.tm_mday; - dts_local.month = tm_.tm_mon + 1; - dts_local.year = tm_.tm_year + 1900; - - /* Extract the timezone offset that was applied */ - rawtime /= 60; - localrawtime = (time_t)get_datetimestruct_days(&dts_local) * 24 * 60; - localrawtime += dts_local.hour * 60; - localrawtime += dts_local.min; - - timezone_offset = localrawtime - rawtime; - - /* Set dts to point to our local time instead of the UTC time */ - dts = &dts_local; - } - /* Use the manually provided tzoffset */ - else if (local) { - /* Make a copy of the npy_datetimestruct we can modify */ - dts_local = *dts; - dts = &dts_local; - - /* Set and apply the required timezone offset */ - timezone_offset = tzoffset; - add_minutes_to_datetimestruct(dts, timezone_offset); - } - - /* YEAR */ -#ifdef _WIN32 - tmplen = _snprintf(substr, sublen, "%04" NPY_INT64_FMT, dts->year); -#else - tmplen = snprintf(substr, sublen, "%04" NPY_INT64_FMT, dts->year); -#endif - /* If it ran out of space or there isn't space for the NULL terminator */ - if (tmplen < 0 || tmplen >= sublen) { - goto string_too_short; - } - substr += tmplen; - sublen -= tmplen; - - /* Stop if the unit is years */ - if (base == NPY_FR_Y) { - *substr = '\0'; - return 0; - } - - /* MONTH */ - substr[0] = '-'; - if (sublen <= 1 ) { - goto string_too_short; - } - substr[1] = (char)((dts->month / 10) + '0'); - if (sublen <= 2 ) { - goto string_too_short; - } - substr[2] = (char)((dts->month % 10) + '0'); - if (sublen <= 3 ) { - goto string_too_short; - } - substr += 3; - sublen -= 3; - - /* Stop if the unit is months */ - if (base == NPY_FR_M) { - *substr = '\0'; - return 0; - } - - /* DAY */ - substr[0] = '-'; - if (sublen <= 1 ) { - goto string_too_short; - } - substr[1] = (char)((dts->day / 10) + '0'); - if (sublen <= 2 ) { - goto string_too_short; - } - substr[2] = (char)((dts->day % 10) + '0'); - if (sublen <= 3 ) { - goto string_too_short; - } - substr += 3; - sublen -= 3; - - /* Stop if the unit is days */ - if (base == NPY_FR_D) { - *substr = '\0'; - return 0; - } - - /* HOUR */ - substr[0] = 'T'; - if (sublen <= 1 ) { - goto string_too_short; - } - substr[1] = (char)((dts->hour / 10) + '0'); - if (sublen <= 2 ) { - goto string_too_short; - } - substr[2] = (char)((dts->hour % 10) + '0'); - if (sublen <= 3 ) { - goto string_too_short; - } - substr += 3; - sublen -= 3; - - /* Stop if the unit is hours */ - if (base == NPY_FR_h) { - goto add_time_zone; - } - - /* MINUTE */ - substr[0] = ':'; - if (sublen <= 1 ) { - goto string_too_short; - } - substr[1] = (char)((dts->min / 10) + '0'); - if (sublen <= 2 ) { - goto string_too_short; - } - substr[2] = (char)((dts->min % 10) + '0'); - if (sublen <= 3 ) { - goto string_too_short; - } - substr += 3; - sublen -= 3; - - /* Stop if the unit is minutes */ - if (base == NPY_FR_m) { - goto add_time_zone; - } - - /* SECOND */ - substr[0] = ':'; - if (sublen <= 1 ) { - goto string_too_short; - } - substr[1] = (char)((dts->sec / 10) + '0'); - if (sublen <= 2 ) { - goto string_too_short; - } - substr[2] = (char)((dts->sec % 10) + '0'); - if (sublen <= 3 ) { - goto string_too_short; - } - substr += 3; - sublen -= 3; - - /* Stop if the unit is seconds */ - if (base == NPY_FR_s) { - goto add_time_zone; - } - - /* MILLISECOND */ - substr[0] = '.'; - if (sublen <= 1 ) { - goto string_too_short; - } - substr[1] = (char)((dts->us / 100000) % 10 + '0'); - if (sublen <= 2 ) { - goto string_too_short; - } - substr[2] = (char)((dts->us / 10000) % 10 + '0'); - if (sublen <= 3 ) { - goto string_too_short; - } - substr[3] = (char)((dts->us / 1000) % 10 + '0'); - if (sublen <= 4 ) { - goto string_too_short; - } - substr += 4; - sublen -= 4; - - /* Stop if the unit is milliseconds */ - if (base == NPY_FR_ms) { - goto add_time_zone; - } - - /* MICROSECOND */ - substr[0] = (char)((dts->us / 100) % 10 + '0'); - if (sublen <= 1 ) { - goto string_too_short; - } - substr[1] = (char)((dts->us / 10) % 10 + '0'); - if (sublen <= 2 ) { - goto string_too_short; - } - substr[2] = (char)(dts->us % 10 + '0'); - if (sublen <= 3 ) { - goto string_too_short; - } - substr += 3; - sublen -= 3; - - /* Stop if the unit is microseconds */ - if (base == NPY_FR_us) { - goto add_time_zone; - } - - /* NANOSECOND */ - substr[0] = (char)((dts->ps / 100000) % 10 + '0'); - if (sublen <= 1 ) { - goto string_too_short; - } - substr[1] = (char)((dts->ps / 10000) % 10 + '0'); - if (sublen <= 2 ) { - goto string_too_short; - } - substr[2] = (char)((dts->ps / 1000) % 10 + '0'); - if (sublen <= 3 ) { - goto string_too_short; - } - substr += 3; - sublen -= 3; - - /* Stop if the unit is nanoseconds */ - if (base == NPY_FR_ns) { - goto add_time_zone; - } - - /* PICOSECOND */ - substr[0] = (char)((dts->ps / 100) % 10 + '0'); - if (sublen <= 1 ) { - goto string_too_short; - } - substr[1] = (char)((dts->ps / 10) % 10 + '0'); - if (sublen <= 2 ) { - goto string_too_short; - } - substr[2] = (char)(dts->ps % 10 + '0'); - if (sublen <= 3 ) { - goto string_too_short; - } - substr += 3; - sublen -= 3; - - /* Stop if the unit is picoseconds */ - if (base == NPY_FR_ps) { - goto add_time_zone; - } - - /* FEMTOSECOND */ - substr[0] = (char)((dts->as / 100000) % 10 + '0'); - if (sublen <= 1 ) { - goto string_too_short; - } - substr[1] = (char)((dts->as / 10000) % 10 + '0'); - if (sublen <= 2 ) { - goto string_too_short; - } - substr[2] = (char)((dts->as / 1000) % 10 + '0'); - if (sublen <= 3 ) { - goto string_too_short; - } - substr += 3; - sublen -= 3; - - /* Stop if the unit is femtoseconds */ - if (base == NPY_FR_fs) { - goto add_time_zone; - } - - /* ATTOSECOND */ - substr[0] = (char)((dts->as / 100) % 10 + '0'); - if (sublen <= 1 ) { - goto string_too_short; - } - substr[1] = (char)((dts->as / 10) % 10 + '0'); - if (sublen <= 2 ) { - goto string_too_short; - } - substr[2] = (char)(dts->as % 10 + '0'); - if (sublen <= 3 ) { - goto string_too_short; - } - substr += 3; - sublen -= 3; - -add_time_zone: - if (local) { - /* Add the +/- sign */ - if (timezone_offset < 0) { - substr[0] = '-'; - timezone_offset = -timezone_offset; - } - else { - substr[0] = '+'; - } - if (sublen <= 1) { - goto string_too_short; - } - substr += 1; - sublen -= 1; - - /* Add the timezone offset */ - substr[0] = (char)((timezone_offset / (10*60)) % 10 + '0'); - if (sublen <= 1 ) { - goto string_too_short; - } - substr[1] = (char)((timezone_offset / 60) % 10 + '0'); - if (sublen <= 2 ) { - goto string_too_short; - } - substr[2] = (char)(((timezone_offset % 60) / 10) % 10 + '0'); - if (sublen <= 3 ) { - goto string_too_short; - } - substr[3] = (char)((timezone_offset % 60) % 10 + '0'); - if (sublen <= 4 ) { - goto string_too_short; - } - substr += 4; - sublen -= 4; - } - /* UTC "Zulu" time */ - else { - substr[0] = 'Z'; - if (sublen <= 1) { - goto string_too_short; - } - substr += 1; - sublen -= 1; - } - - /* Add a NULL terminator, and return */ - substr[0] = '\0'; - - return 0; - -string_too_short: - /* Put a NULL terminator on anyway */ - if (outlen > 0) { - outstr[outlen-1] = '\0'; - } - - PyErr_Format(PyExc_RuntimeError, - "The string provided for NumPy ISO datetime formatting " - "was too short, with length %d", - outlen); - return -1; -} - -/* - * Tests for and converts a Python datetime.datetime or datetime.date - * object into a NumPy npy_datetimestruct. - * - * While the C API has PyDate_* and PyDateTime_* functions, the following - * implementation just asks for attributes, and thus supports - * datetime duck typing. The tzinfo time zone conversion would require - * this style of access anyway. - * - * 'out_bestunit' gives a suggested unit based on whether the object - * was a datetime.date or datetime.datetime object. - * - * Returns -1 on error, 0 on success, and 1 (with no error set) - * if obj doesn't have the neeeded date or datetime attributes. - */ -NPY_NO_EXPORT int -convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, - NPY_DATETIMEUNIT *out_bestunit) -{ - PyObject *tmp; - int isleap; + PyObject *tmp; + int isleap; /* Initialize the output to all zeros */ memset(out, 0, sizeof(npy_datetimestruct)); @@ -3555,7 +2298,8 @@ convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, goto invalid_date; } isleap = is_leapyear(out->year); - if (out->day < 1 || out->day > days_in_month[isleap][out->month-1]) { + if (out->day < 1 || + out->day > _days_per_month_table[isleap][out->month-1]) { goto invalid_date; } @@ -3627,7 +2371,7 @@ convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, } /* Apply the time zone offset if it exists */ - if (PyObject_HasAttrString(obj, "tzinfo")) { + if (apply_tzinfo && PyObject_HasAttrString(obj, "tzinfo")) { tmp = PyObject_GetAttrString(obj, "tzinfo"); if (tmp == NULL) { return -1; @@ -3648,10 +2392,10 @@ convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, Py_DECREF(tmp); /* - * The timedelta should have an attribute "seconds" + * The timedelta should have a function "total_seconds" * which contains the value we want. */ - tmp = PyObject_GetAttrString(obj, "seconds"); + tmp = PyObject_CallMethod(offset, "total_seconds", ""); if (tmp == NULL) { return -1; } @@ -3690,6 +2434,43 @@ convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, return -1; } +/* + * Gets a tzoffset in minutes by calling the fromutc() function on + * the Python datetime.tzinfo object. + */ +NPY_NO_EXPORT int +get_tzoffset_from_pytzinfo(PyObject *timezone, npy_datetimestruct *dts) +{ + PyObject *dt, *loc_dt; + npy_datetimestruct loc_dts; + + /* Create a Python datetime to give to the timezone object */ + dt = PyDateTime_FromDateAndTime((int)dts->year, dts->month, dts->day, + dts->hour, dts->min, 0, 0); + if (dt == NULL) { + return -1; + } + + /* Convert the datetime from UTC to local time */ + loc_dt = PyObject_CallMethod(timezone, "fromutc", "O", dt); + Py_DECREF(dt); + if (loc_dt == NULL) { + return -1; + } + + /* Convert the local datetime into a datetimestruct */ + if (convert_pydatetime_to_datetimestruct(loc_dt, &loc_dts, NULL, 0) < 0) { + Py_DECREF(loc_dt); + return -1; + } + + Py_DECREF(loc_dt); + + /* Calculate the tzoffset as the difference between the datetimes */ + return get_datetimestruct_minutes(&loc_dts) - + get_datetimestruct_minutes(dts); +} + /* * Converts a PyObject * into a datetime, in any of the forms supported. * @@ -3697,11 +2478,17 @@ convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, * to -1, and this function will populate meta with either default * values or values from the input object. * + * The 'casting' parameter is used to control what kinds of inputs + * are accepted, and what happens. For example, with 'unsafe' casting, + * unrecognized inputs are converted to 'NaT' instead of throwing an error, + * while with 'safe' casting an error will be thrown if any precision + * from the input will be thrown away. + * * Returns -1 on error, 0 on success. */ NPY_NO_EXPORT int convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, - npy_datetime *out) + NPY_CASTING casting, npy_datetime *out) { if (PyBytes_Check(obj) || PyUnicode_Check(obj)) { PyObject *bytes = NULL; @@ -3727,8 +2514,8 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, } /* Parse the ISO date */ - if (parse_iso_8601_date(str, len, meta->base, &dts, - NULL, &bestunit, NULL) < 0) { + if (parse_iso_8601_datetime(str, len, meta->base, casting, + &dts, NULL, &bestunit, NULL) < 0) { Py_DECREF(bytes); return -1; } @@ -3738,7 +2525,6 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, if (meta->base == -1) { meta->base = bestunit; meta->num = 1; - meta->events = 1; } if (convert_datetimestruct_to_datetime(meta, &dts, out) < 0) { @@ -3758,31 +2544,6 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, *out = PyLong_AsLongLong(obj); return 0; } - /* Could be a tuple with event number in the second entry */ - else if (PyTuple_Check(obj) && PyTuple_Size(obj) == 2) { - int event, event_old; - if (convert_pyobject_to_datetime(meta, PyTuple_GET_ITEM(obj, 0), - out) < 0) { - return -1; - } - event = (int)PyInt_AsLong(PyTuple_GET_ITEM(obj, 1)); - if (event == -1 && PyErr_Occurred()) { - return -1; - } - if (event < 0 || event >= meta->events) { - PyErr_SetString(PyExc_ValueError, "event value for NumPy " - "datetime is out of range"); - return -1; - } - /* Replace the event with the one from the tuple */ - event_old = *out % meta->events; - if (event_old < 0) { - event_old += meta->events; - } - *out = *out - event_old + event; - - return 0; - } /* Datetime scalar */ else if (PyArray_IsScalar(obj, Datetime)) { PyDatetimeScalarObject *dts = (PyDatetimeScalarObject *)obj; @@ -3796,8 +2557,17 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, } /* Otherwise do a casting transformation */ else { - return cast_datetime_to_datetime(&dts->obmeta, meta, - dts->obval, out); + /* Allow NaT (not-a-time) values to slip through any rule */ + if (dts->obval != NPY_DATETIME_NAT && + raise_if_datetime64_metadata_cast_error( + "NumPy timedelta64 scalar", + &dts->obmeta, meta, casting) < 0) { + return -1; + } + else { + return cast_datetime_to_datetime(&dts->obmeta, meta, + dts->obval, out); + } } } /* Datetime zero-dimensional array */ @@ -3825,7 +2595,16 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, } /* Otherwise do a casting transformation */ else { - return cast_datetime_to_datetime(obj_meta, meta, dt, out); + /* Allow NaT (not-a-time) values to slip through any rule */ + if (dt != NPY_DATETIME_NAT && + raise_if_datetime64_metadata_cast_error( + "NumPy timedelta64 scalar", + obj_meta, meta, casting) < 0) { + return -1; + } + else { + return cast_datetime_to_datetime(obj_meta, meta, dt, out); + } } } /* Convert from a Python date or datetime object */ @@ -3834,7 +2613,7 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, npy_datetimestruct dts; NPY_DATETIMEUNIT bestunit = -1; - code = convert_pydatetime_to_datetimestruct(obj, &dts, &bestunit); + code = convert_pydatetime_to_datetimestruct(obj, &dts, &bestunit, 1); if (code == -1) { return -1; } @@ -3843,20 +2622,42 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, if (meta->base == -1) { meta->base = bestunit; meta->num = 1; - meta->events = 1; } - - if (convert_datetimestruct_to_datetime(meta, &dts, out) < 0) { - return -1; + else { + PyArray_DatetimeMetaData obj_meta; + obj_meta.base = bestunit; + obj_meta.num = 1; + + if (raise_if_datetime64_metadata_cast_error( + bestunit == NPY_FR_D ? "datetime.date object" + : "datetime.datetime object", + &obj_meta, meta, casting) < 0) { + return -1; + } } - return 0; + return convert_datetimestruct_to_datetime(meta, &dts, out); } } - PyErr_SetString(PyExc_ValueError, - "Could not convert object to NumPy datetime"); - return -1; + /* + * With unsafe casting, convert unrecognized objects into NaT + * and with same_kind casting, convert None into NaT + */ + if (casting == NPY_UNSAFE_CASTING || + (obj == Py_None && casting == NPY_SAME_KIND_CASTING)) { + if (meta->base == -1) { + meta->base = NPY_FR_GENERIC; + meta->num = 1; + } + *out = NPY_DATETIME_NAT; + return 0; + } + else { + PyErr_SetString(PyExc_ValueError, + "Could not convert object to NumPy datetime"); + return -1; + } } /* @@ -3866,19 +2667,74 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, * to -1, and this function will populate meta with either default * values or values from the input object. * + * The 'casting' parameter is used to control what kinds of inputs + * are accepted, and what happens. For example, with 'unsafe' casting, + * unrecognized inputs are converted to 'NaT' instead of throwing an error, + * while with 'safe' casting an error will be thrown if any precision + * from the input will be thrown away. + * * Returns -1 on error, 0 on success. */ NPY_NO_EXPORT int convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj, - npy_timedelta *out) + NPY_CASTING casting, npy_timedelta *out) { + if (PyBytes_Check(obj) || PyUnicode_Check(obj)) { + PyObject *bytes = NULL; + char *str = NULL; + Py_ssize_t len = 0; + int succeeded = 0; + + /* Convert to an ASCII string for the date parser */ + if (PyUnicode_Check(obj)) { + bytes = PyUnicode_AsASCIIString(obj); + if (bytes == NULL) { + return -1; + } + } + else { + bytes = obj; + Py_INCREF(bytes); + } + if (PyBytes_AsStringAndSize(bytes, &str, &len) == -1) { + Py_DECREF(bytes); + return -1; + } + + /* Check for a NaT string */ + if (len <= 0 || (len == 3 && + tolower(str[0]) == 'n' && + tolower(str[1]) == 'a' && + tolower(str[2]) == 't')) { + *out = NPY_DATETIME_NAT; + succeeded = 1; + } + /* Parse as an integer */ + else { + char *strend = NULL; + + *out = strtol(str, &strend, 10); + if (strend - str == len) { + succeeded = 1; + } + } + + if (succeeded) { + /* Use generic units if none was specified */ + if (meta->base == -1) { + meta->base = NPY_FR_GENERIC; + meta->num = 1; + } + + return 0; + } + } /* Do no conversion on raw integers */ - if (PyInt_Check(obj) || PyLong_Check(obj)) { + else if (PyInt_Check(obj) || PyLong_Check(obj)) { /* Use the default unit if none was specified */ if (meta->base == -1) { meta->base = NPY_DATETIME_DEFAULTUNIT; meta->num = 1; - meta->events = 1; } *out = PyLong_AsLongLong(obj); @@ -3897,8 +2753,17 @@ convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj, } /* Otherwise do a casting transformation */ else { - return cast_timedelta_to_timedelta(&dts->obmeta, meta, - dts->obval, out); + /* Allow NaT (not-a-time) values to slip through any rule */ + if (dts->obval != NPY_DATETIME_NAT && + raise_if_timedelta64_metadata_cast_error( + "NumPy timedelta64 scalar", + &dts->obmeta, meta, casting) < 0) { + return -1; + } + else { + return cast_timedelta_to_timedelta(&dts->obmeta, meta, + dts->obval, out); + } } } /* Timedelta zero-dimensional array */ @@ -3926,7 +2791,16 @@ convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj, } /* Otherwise do a casting transformation */ else { - return cast_timedelta_to_timedelta(obj_meta, meta, dt, out); + /* Allow NaT (not-a-time) values to slip through any rule */ + if (dt != NPY_DATETIME_NAT && + raise_if_timedelta64_metadata_cast_error( + "NumPy timedelta64 scalar", + obj_meta, meta, casting) < 0) { + return -1; + } + else { + return cast_timedelta_to_timedelta(obj_meta, meta, dt, out); + } } } /* Convert from a Python timedelta object */ @@ -3981,7 +2855,6 @@ convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj, if (meta->base == -1) { meta->base = NPY_FR_us; meta->num = 1; - meta->events = 1; *out = td; @@ -3989,20 +2862,62 @@ convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj, } else { /* - * Convert to a microseconds timedelta, then cast to the - * desired units. + * Detect the largest unit where every value after is zero, + * to allow safe casting to seconds if microseconds is zero, + * for instance. */ - us_meta.base = NPY_FR_us; + if (td % 1000LL != 0) { + us_meta.base = NPY_FR_us; + } + else if (td % 1000000LL != 0) { + us_meta.base = NPY_FR_ms; + } + else if (td % (60*1000000LL) != 0) { + us_meta.base = NPY_FR_s; + } + else if (td % (60*60*1000000LL) != 0) { + us_meta.base = NPY_FR_m; + } + else if (td % (24*60*60*1000000LL) != 0) { + us_meta.base = NPY_FR_D; + } + else if (td % (7*24*60*60*1000000LL) != 0) { + us_meta.base = NPY_FR_W; + } us_meta.num = 1; - us_meta.events = 1; - - return cast_timedelta_to_timedelta(&us_meta, meta, td, out); + + if (raise_if_timedelta64_metadata_cast_error( + "datetime.timedelta object", + &us_meta, meta, casting) < 0) { + return -1; + } + else { + /* Switch back to microseconds for the casting operation */ + us_meta.base = NPY_FR_us; + + return cast_timedelta_to_timedelta(&us_meta, meta, td, out); + } } } - PyErr_SetString(PyExc_ValueError, - "Could not convert object to NumPy timedelta"); - return -1; + /* + * With unsafe casting, convert unrecognized objects into NaT + * and with same_kind casting, convert None into NaT + */ + if (casting == NPY_UNSAFE_CASTING || + (obj == Py_None && casting == NPY_SAME_KIND_CASTING)) { + if (meta->base == -1) { + meta->base = NPY_FR_GENERIC; + meta->num = 1; + } + *out = NPY_DATETIME_NAT; + return 0; + } + else { + PyErr_SetString(PyExc_ValueError, + "Could not convert object to NumPy timedelta"); + return -1; + } } /* @@ -4016,17 +2931,20 @@ convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj, NPY_NO_EXPORT PyObject * convert_datetime_to_pyobject(npy_datetime dt, PyArray_DatetimeMetaData *meta) { - PyObject *ret = NULL, *tup = NULL; + PyObject *ret = NULL; npy_datetimestruct dts; - /* Handle not-a-time, and generic units as NaT as well */ + /* + * Convert NaT (not-a-time) and any value with generic units + * into None. + */ if (dt == NPY_DATETIME_NAT || meta->base == NPY_FR_GENERIC) { - return PyUString_FromString("NaT"); + Py_INCREF(Py_None); + return Py_None; } /* If the type's precision is greater than microseconds, return an int */ if (meta->base > NPY_FR_us) { - /* Skip use of a tuple for the events, just return the raw int */ return PyLong_FromLongLong(dt); } @@ -4041,7 +2959,6 @@ convert_datetime_to_pyobject(npy_datetime dt, PyArray_DatetimeMetaData *meta) * return a raw int. */ if (dts.year < 1 || dts.year > 9999 || dts.sec == 60) { - /* Also skip use of a tuple for the events */ return PyLong_FromLongLong(dt); } @@ -4055,28 +2972,7 @@ convert_datetime_to_pyobject(npy_datetime dt, PyArray_DatetimeMetaData *meta) ret = PyDate_FromDate(dts.year, dts.month, dts.day); } - /* If there is one event, just return the datetime */ - if (meta->events == 1) { - return ret; - } - /* Otherwise return a tuple with the event in the second position */ - else { - tup = PyTuple_New(2); - if (tup == NULL) { - Py_DECREF(ret); - return NULL; - } - PyTuple_SET_ITEM(tup, 0, ret); - - ret = PyInt_FromLong(dts.event); - if (ret == NULL) { - Py_DECREF(tup); - return NULL; - } - PyTuple_SET_ITEM(tup, 1, ret); - - return tup; - } + return ret; } /* @@ -4089,14 +2985,16 @@ convert_datetime_to_pyobject(npy_datetime dt, PyArray_DatetimeMetaData *meta) NPY_NO_EXPORT PyObject * convert_timedelta_to_pyobject(npy_timedelta td, PyArray_DatetimeMetaData *meta) { - PyObject *ret = NULL, *tup = NULL; + PyObject *ret = NULL; npy_timedelta value; - int event = 0; int days = 0, seconds = 0, useconds = 0; - /* Handle not-a-time */ + /* + * Convert NaT (not-a-time) into None. + */ if (td == NPY_DATETIME_NAT) { - return PyUString_FromString("NaT"); + Py_INCREF(Py_None); + return Py_None; } /* @@ -4107,22 +3005,11 @@ convert_timedelta_to_pyobject(npy_timedelta td, PyArray_DatetimeMetaData *meta) meta->base == NPY_FR_Y || meta->base == NPY_FR_M || meta->base == NPY_FR_GENERIC) { - /* Skip use of a tuple for the events, just return the raw int */ return PyLong_FromLongLong(td); } value = td; - /* If there are events, extract the event */ - if (meta->events > 1) { - event = (int)(value % meta->events); - value = value / meta->events; - if (event < 0) { - --value; - event += meta->events; - } - } - /* Apply the unit multiplier (TODO: overflow treatment...) */ value *= meta->num; @@ -4176,28 +3063,7 @@ convert_timedelta_to_pyobject(npy_timedelta td, PyArray_DatetimeMetaData *meta) } } - /* If there is one event, just return the datetime */ - if (meta->events == 1) { - return ret; - } - /* Otherwise return a tuple with the event in the second position */ - else { - tup = PyTuple_New(2); - if (tup == NULL) { - Py_DECREF(ret); - return NULL; - } - PyTuple_SET_ITEM(tup, 0, ret); - - ret = PyInt_FromLong(event); - if (ret == NULL) { - Py_DECREF(tup); - return NULL; - } - PyTuple_SET_ITEM(tup, 1, ret); - - return tup; - } + return ret; } /* @@ -4226,14 +3092,13 @@ has_equivalent_datetime_metadata(PyArray_Descr *type1, PyArray_Descr *type2) return 0; } - /* For generic units, the num and events are ignored */ + /* For generic units, the num is ignored */ if (meta1->base == NPY_FR_GENERIC && meta2->base == NPY_FR_GENERIC) { return 1; } return meta1->base == meta2->base && - meta1->num == meta2->num && - meta1->events == meta2->events; + meta1->num == meta2->num; } /* @@ -4252,8 +3117,7 @@ cast_datetime_to_datetime(PyArray_DatetimeMetaData *src_meta, /* If the metadata is the same, short-circuit the conversion */ if (src_meta->base == dst_meta->base && - src_meta->num == dst_meta->num && - src_meta->events == dst_meta->events) { + src_meta->num == dst_meta->num) { *dst_dt = src_dt; return 0; } @@ -4263,9 +3127,6 @@ cast_datetime_to_datetime(PyArray_DatetimeMetaData *src_meta, *dst_dt = NPY_DATETIME_NAT; return -1; } - if (dts.event >= dst_meta->events) { - dts.event = dts.event % dst_meta->events; - } if (convert_datetimestruct_to_datetime(dst_meta, &dts, dst_dt) < 0) { *dst_dt = NPY_DATETIME_NAT; return -1; @@ -4287,12 +3148,10 @@ cast_timedelta_to_timedelta(PyArray_DatetimeMetaData *src_meta, npy_timedelta *dst_dt) { npy_int64 num = 0, denom = 0; - int event = 0; /* If the metadata is the same, short-circuit the conversion */ if (src_meta->base == dst_meta->base && - src_meta->num == dst_meta->num && - src_meta->events == dst_meta->events) { + src_meta->num == dst_meta->num) { *dst_dt = src_dt; return 0; } @@ -4304,16 +3163,6 @@ cast_timedelta_to_timedelta(PyArray_DatetimeMetaData *src_meta, return -1; } - /* Remove the event number from the value */ - if (src_meta->events > 1) { - event = (int)(src_dt % src_meta->events); - src_dt = src_dt / src_meta->events; - if (event < 0) { - --src_dt; - event += src_meta->events; - } - } - /* Apply the scaling */ if (src_dt < 0) { *dst_dt = (src_dt * num - (denom - 1)) / denom; @@ -4322,12 +3171,6 @@ cast_timedelta_to_timedelta(PyArray_DatetimeMetaData *src_meta, *dst_dt = src_dt * num / denom; } - /* Add the event number back in */ - if (dst_meta->events > 1) { - event = event % dst_meta->events; - *dst_dt = (*dst_dt) * dst_meta->events + event; - } - return 0; } @@ -4385,6 +3228,7 @@ is_any_numpy_datetime_or_timedelta(PyObject *obj) NPY_NO_EXPORT int convert_pyobjects_to_datetimes(int count, PyObject **objs, int *type_nums, + NPY_CASTING casting, npy_int64 *out_values, PyArray_DatetimeMetaData *inout_meta) { @@ -4409,7 +3253,6 @@ convert_pyobjects_to_datetimes(int count, for (i = 0; i < count; ++i) { meta[i].base = -1; meta[i].num = 1; - meta[i].events = 1; /* NULL -> NaT */ if (objs[i] == NULL) { @@ -4418,14 +3261,14 @@ convert_pyobjects_to_datetimes(int count, } else if (type_nums[i] == NPY_DATETIME) { if (convert_pyobject_to_datetime(&meta[i], objs[i], - &out_values[i]) < 0) { + casting, &out_values[i]) < 0) { PyArray_free(meta); return -1; } } else if (type_nums[i] == NPY_TIMEDELTA) { if (convert_pyobject_to_timedelta(&meta[i], objs[i], - &out_values[i]) < 0) { + casting, &out_values[i]) < 0) { PyArray_free(meta); return -1; } @@ -4484,13 +3327,13 @@ convert_pyobjects_to_datetimes(int count, } else if (type_nums[i] == NPY_DATETIME) { if (convert_pyobject_to_datetime(inout_meta, objs[i], - &out_values[i]) < 0) { + casting, &out_values[i]) < 0) { return -1; } } else if (type_nums[i] == NPY_TIMEDELTA) { if (convert_pyobject_to_timedelta(inout_meta, objs[i], - &out_values[i]) < 0) { + casting, &out_values[i]) < 0) { return -1; } } @@ -4623,7 +3466,7 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, /* Convert all the arguments */ if (convert_pyobjects_to_datetimes(3, objs, type_nums, - values, &meta) < 0) { + NPY_SAME_KIND_CASTING, values, &meta) < 0) { return NULL; } @@ -4699,6 +3542,132 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, return ret; } +/* + * Examines all the strings in the given string array, and parses them + * to find the right metadata. + * + * Returns 0 on success, -1 on failure. + */ +static int +find_string_array_datetime64_type(PyObject *obj, + PyArray_DatetimeMetaData *meta) +{ + NpyIter* iter; + NpyIter_IterNextFunc *iternext; + char **dataptr; + npy_intp *strideptr, *innersizeptr; + PyArray_Descr *string_dtype; + int maxlen; + char *tmp_buffer = NULL; + + npy_datetimestruct dts; + PyArray_DatetimeMetaData tmp_meta; + + /* Handle zero-sized arrays specially */ + if (PyArray_SIZE(obj) == 0) { + return 0; + } + + string_dtype = PyArray_DescrFromType(NPY_STRING); + if (string_dtype == NULL) { + return -1; + } + + /* Use unsafe casting to allow unicode -> ascii string */ + iter = NpyIter_New((PyArrayObject *)obj, + NPY_ITER_READONLY| + NPY_ITER_EXTERNAL_LOOP| + NPY_ITER_BUFFERED, + NPY_KEEPORDER, NPY_UNSAFE_CASTING, + string_dtype); + Py_DECREF(string_dtype); + if (iter == NULL) { + return -1; + } + + iternext = NpyIter_GetIterNext(iter, NULL); + if (iternext == NULL) { + NpyIter_Deallocate(iter); + return -1; + } + dataptr = NpyIter_GetDataPtrArray(iter); + strideptr = NpyIter_GetInnerStrideArray(iter); + innersizeptr = NpyIter_GetInnerLoopSizePtr(iter); + + /* Get the resulting string length */ + maxlen = NpyIter_GetDescrArray(iter)[0]->elsize; + + /* Allocate a buffer for strings which fill the buffer completely */ + tmp_buffer = PyArray_malloc(maxlen+1); + if (tmp_buffer == NULL) { + PyErr_NoMemory(); + NpyIter_Deallocate(iter); + return -1; + } + + /* The iteration loop */ + do { + /* Get the inner loop data/stride/count values */ + char* data = *dataptr; + npy_intp stride = *strideptr; + npy_intp count = *innersizeptr; + char *tmp; + + /* The inner loop */ + while (count--) { + /* Replicating strnlen with memchr, because Mac OS X lacks it */ + tmp = memchr(data, '\0', maxlen); + + /* If the string is all full, use the buffer */ + if (tmp == NULL) { + memcpy(tmp_buffer, data, maxlen); + tmp_buffer[maxlen] = '\0'; + + tmp_meta.base = -1; + if (parse_iso_8601_datetime(tmp_buffer, maxlen, -1, + NPY_UNSAFE_CASTING, &dts, NULL, + &tmp_meta.base, NULL) < 0) { + goto fail; + } + } + /* Otherwise parse the data in place */ + else { + tmp_meta.base = -1; + if (parse_iso_8601_datetime(data, tmp - data, -1, + NPY_UNSAFE_CASTING, &dts, NULL, + &tmp_meta.base, NULL) < 0) { + goto fail; + } + } + + tmp_meta.num = 1; + /* Combine it with 'meta' */ + if (compute_datetime_metadata_greatest_common_divisor(meta, + &tmp_meta, meta, 0, 0) < 0) { + goto fail; + } + + + data += stride; + } + } while(iternext(iter)); + + PyArray_free(tmp_buffer); + NpyIter_Deallocate(iter); + + return 0; + +fail: + if (tmp_buffer != NULL) { + PyArray_free(tmp_buffer); + } + if (iter != NULL) { + NpyIter_Deallocate(iter); + } + + return -1; +} + /* * Recursively determines the metadata for an NPY_DATETIME dtype. @@ -4712,8 +3681,13 @@ recursive_find_object_datetime64_type(PyObject *obj, /* Array -> use its metadata */ if (PyArray_Check(obj)) { PyArray_Descr *obj_dtype = PyArray_DESCR(obj); + + if (obj_dtype->type_num == NPY_STRING || + obj_dtype->type_num == NPY_UNICODE) { + return find_string_array_datetime64_type(obj, meta); + } /* If the array has metadata, use it */ - if (obj_dtype->type_num == NPY_DATETIME || + else if (obj_dtype->type_num == NPY_DATETIME || obj_dtype->type_num == NPY_TIMEDELTA) { PyArray_DatetimeMetaData *tmp_meta; @@ -4755,9 +3729,9 @@ recursive_find_object_datetime64_type(PyObject *obj, tmp_meta.base = -1; tmp_meta.num = 1; - tmp_meta.events = 1; - if (convert_pyobject_to_datetime(&tmp_meta, obj, &tmp) < 0) { + if (convert_pyobject_to_datetime(&tmp_meta, obj, + NPY_UNSAFE_CASTING, &tmp) < 0) { /* If it's a value error, clear the error */ if (PyErr_Occurred() && PyErr_GivenExceptionMatches(PyErr_Occurred(), @@ -4785,7 +3759,6 @@ recursive_find_object_datetime64_type(PyObject *obj, tmp_meta.base = NPY_FR_D; tmp_meta.num = 1; - tmp_meta.events = 1; /* Combine it with 'meta' */ if (compute_datetime_metadata_greatest_common_divisor(meta, @@ -4801,7 +3774,6 @@ recursive_find_object_datetime64_type(PyObject *obj, tmp_meta.base = NPY_FR_us; tmp_meta.num = 1; - tmp_meta.events = 1; /* Combine it with 'meta' */ if (compute_datetime_metadata_greatest_common_divisor(meta, @@ -4855,6 +3827,7 @@ recursive_find_object_timedelta64_type(PyObject *obj, /* Array -> use its metadata */ if (PyArray_Check(obj)) { PyArray_Descr *obj_dtype = PyArray_DESCR(obj); + /* If the array has metadata, use it */ if (obj_dtype->type_num == NPY_DATETIME || obj_dtype->type_num == NPY_TIMEDELTA) { @@ -4902,7 +3875,6 @@ recursive_find_object_timedelta64_type(PyObject *obj, tmp_meta.base = NPY_FR_us; tmp_meta.num = 1; - tmp_meta.events = 1; /* Combine it with 'meta' */ if (compute_datetime_metadata_greatest_common_divisor(meta, @@ -4956,7 +3928,6 @@ find_object_datetime_type(PyObject *obj, int type_num) meta.base = NPY_FR_GENERIC; meta.num = 1; - meta.events = 1; if (type_num == NPY_DATETIME) { if (recursive_find_object_datetime64_type(obj, &meta) < 0) { diff --git a/numpy/core/src/multiarray/datetime_busday.c b/numpy/core/src/multiarray/datetime_busday.c index 2d83786a6b4b..67e403cf7ca0 100644 --- a/numpy/core/src/multiarray/datetime_busday.c +++ b/numpy/core/src/multiarray/datetime_busday.c @@ -465,7 +465,6 @@ business_day_offset(PyArrayObject *dates, PyArrayObject *offsets, /* First create the data types for dates and offsets */ temp_meta.base = NPY_FR_D; temp_meta.num = 1; - temp_meta.events = 1; dtypes[0] = create_datetime_dtype(NPY_DATETIME, &temp_meta); if (dtypes[0] == NULL) { goto fail; @@ -599,7 +598,6 @@ business_day_count(PyArrayObject *dates_begin, PyArrayObject *dates_end, /* First create the data types for the dates and the int64 output */ temp_meta.base = NPY_FR_D; temp_meta.num = 1; - temp_meta.events = 1; dtypes[0] = create_datetime_dtype(NPY_DATETIME, &temp_meta); if (dtypes[0] == NULL) { goto fail; @@ -730,7 +728,6 @@ is_business_day(PyArrayObject *dates, PyArrayObject *out, /* First create the data types for the dates and the bool output */ temp_meta.base = NPY_FR_D; temp_meta.num = 1; - temp_meta.events = 1; dtypes[0] = create_datetime_dtype(NPY_DATETIME, &temp_meta); if (dtypes[0] == NULL) { goto fail; diff --git a/numpy/core/src/multiarray/datetime_strings.c b/numpy/core/src/multiarray/datetime_strings.c new file mode 100644 index 000000000000..c8cbe85cfd14 --- /dev/null +++ b/numpy/core/src/multiarray/datetime_strings.c @@ -0,0 +1,1521 @@ +/* + * This file implements string parsing and creation for NumPy datetime. + * + * Written by Mark Wiebe (mwwiebe@gmail.com) + * Copyright (c) 2011 by Enthought, Inc. + * + * See LICENSE.txt for the license. + */ + +#define PY_SSIZE_T_CLEAN +#include + +#include + +#define _MULTIARRAYMODULE +#include + +#include "npy_config.h" +#include "numpy/npy_3kcompat.h" + +#include "numpy/arrayscalars.h" +#include "methods.h" +#include "_datetime.h" +#include "datetime_strings.h" + +/* + * Parses (almost) standard ISO 8601 date strings. The differences are: + * + * + The date "20100312" is parsed as the year 20100312, not as + * equivalent to "2010-03-12". The '-' in the dates are not optional. + * + Only seconds may have a decimal point, with up to 18 digits after it + * (maximum attoseconds precision). + * + Either a 'T' as in ISO 8601 or a ' ' may be used to separate + * the date and the time. Both are treated equivalently. + * + Doesn't (yet) handle the "YYYY-DDD" or "YYYY-Www" formats. + * + Doesn't handle leap seconds (seconds value has 60 in these cases). + * + Doesn't handle 24:00:00 as synonym for midnight (00:00:00) tomorrow + * + Accepts special values "NaT" (not a time), "Today", (current + * day according to local time) and "Now" (current time in UTC). + * + * 'str' must be a NULL-terminated string, and 'len' must be its length. + * 'unit' should contain -1 if the unit is unknown, or the unit + * which will be used if it is. + * 'casting' controls how the detected unit from the string is allowed + * to be cast to the 'unit' parameter. + * + * 'out' gets filled with the parsed date-time. + * 'out_local' gets set to 1 if the parsed time was in local time, + * to 0 otherwise. The values 'now' and 'today' don't get counted + * as local, and neither do UTC +/-#### timezone offsets, because + * they aren't using the computer's local timezone offset. + * 'out_bestunit' gives a suggested unit based on the amount of + * resolution provided in the string, or -1 for NaT. + * 'out_special' gets set to 1 if the parsed time was 'today', + * 'now', or ''/'NaT'. For 'today', the unit recommended is + * 'D', for 'now', the unit recommended is 's', and for 'NaT' + * the unit recommended is 'Y'. + * + * Returns 0 on success, -1 on failure. + */ +NPY_NO_EXPORT int +parse_iso_8601_datetime(char *str, int len, + NPY_DATETIMEUNIT unit, + NPY_CASTING casting, + npy_datetimestruct *out, + npy_bool *out_local, + NPY_DATETIMEUNIT *out_bestunit, + npy_bool *out_special) +{ + int year_leap = 0; + int i, numdigits; + char *substr, sublen; + NPY_DATETIMEUNIT bestunit; + + /* Initialize the output to all zeros */ + memset(out, 0, sizeof(npy_datetimestruct)); + out->month = 1; + out->day = 1; + + /* + * Convert the empty string and case-variants of "NaT" to not-a-time. + * Tried to use PyOS_stricmp, but that function appears to be broken, + * not even matching the strcmp function signature as it should. + */ + if (len <= 0 || (len == 3 && + tolower(str[0]) == 'n' && + tolower(str[1]) == 'a' && + tolower(str[2]) == 't')) { + out->year = NPY_DATETIME_NAT; + + /* + * Indicate that this was a special value, and + * recommend generic units. + */ + if (out_local != NULL) { + *out_local = 0; + } + if (out_bestunit != NULL) { + *out_bestunit = NPY_FR_GENERIC; + } + if (out_special != NULL) { + *out_special = 1; + } + + return 0; + } + + if (unit == NPY_FR_GENERIC) { + PyErr_SetString(PyExc_ValueError, + "Cannot create a NumPy datetime other than NaT " + "with generic units"); + return -1; + } + + /* + * The string "today" means take today's date in local time, and + * convert it to a date representation. This date representation, if + * forced into a time unit, will be at midnight UTC. + * This is perhaps a little weird, but done so that the + * 'datetime64[D]' type produces the date you expect, rather than + * switching to an adjacent day depending on the current time and your + * timezone. + */ + if (len == 5 && tolower(str[0]) == 't' && + tolower(str[1]) == 'o' && + tolower(str[2]) == 'd' && + tolower(str[3]) == 'a' && + tolower(str[4]) == 'y') { + time_t rawtime = 0; + struct tm tm_; + + time(&rawtime); +#if defined(_WIN32) + if (localtime_s(&tm_, &rawtime) != 0) { + PyErr_SetString(PyExc_OSError, "Failed to obtain local time " + "from localtime_s"); + return -1; + } +#else + /* Other platforms may require something else */ + if (localtime_r(&rawtime, &tm_) == NULL) { + PyErr_SetString(PyExc_OSError, "Failed obtain local time " + "from localtime_r"); + return -1; + } +#endif + out->year = tm_.tm_year + 1900; + out->month = tm_.tm_mon + 1; + out->day = tm_.tm_mday; + + bestunit = NPY_FR_D; + + /* + * Indicate that this was a special value, and + * is a date (unit 'D'). + */ + if (out_local != NULL) { + *out_local = 0; + } + if (out_bestunit != NULL) { + *out_bestunit = bestunit; + } + if (out_special != NULL) { + *out_special = 1; + } + + /* Check the casting rule */ + if (unit != -1 && !can_cast_datetime64_units(bestunit, unit, + casting)) { + PyErr_Format(PyExc_TypeError, "Cannot parse \"%s\" as unit " + "'%s' using casting rule %s", + str, _datetime_strings[unit], + npy_casting_to_string(casting)); + return -1; + } + + return 0; + } + + /* The string "now" resolves to the current UTC time */ + if (len == 3 && tolower(str[0]) == 'n' && + tolower(str[1]) == 'o' && + tolower(str[2]) == 'w') { + time_t rawtime = 0; + PyArray_DatetimeMetaData meta; + + time(&rawtime); + + /* Set up a dummy metadata for the conversion */ + meta.base = NPY_FR_s; + meta.num = 1; + + bestunit = NPY_FR_s; + + /* + * Indicate that this was a special value, and + * use 's' because the time() function has resolution + * seconds. + */ + if (out_local != NULL) { + *out_local = 0; + } + if (out_bestunit != NULL) { + *out_bestunit = bestunit; + } + if (out_special != NULL) { + *out_special = 1; + } + + /* Check the casting rule */ + if (unit != -1 && !can_cast_datetime64_units(bestunit, unit, + casting)) { + PyErr_Format(PyExc_TypeError, "Cannot parse \"%s\" as unit " + "'%s' using casting rule %s", + str, _datetime_strings[unit], + npy_casting_to_string(casting)); + return -1; + } + + return convert_datetime_to_datetimestruct(&meta, rawtime, out); + } + + /* Anything else isn't a special value */ + if (out_special != NULL) { + *out_special = 0; + } + + substr = str; + sublen = len; + + /* Skip leading whitespace */ + while (sublen > 0 && isspace(*substr)) { + ++substr; + --sublen; + } + + /* Leading '-' sign for negative year */ + if (*substr == '-') { + ++substr; + --sublen; + } + + if (sublen == 0) { + goto parse_error; + } + + /* PARSE THE YEAR (digits until the '-' character) */ + out->year = 0; + while (sublen > 0 && isdigit(*substr)) { + out->year = 10 * out->year + (*substr - '0'); + ++substr; + --sublen; + } + + /* Negate the year if necessary */ + if (str[0] == '-') { + out->year = -out->year; + } + /* Check whether it's a leap-year */ + year_leap = is_leapyear(out->year); + + /* Next character must be a '-' or the end of the string */ + if (sublen == 0) { + if (out_local != NULL) { + *out_local = 0; + } + bestunit = NPY_FR_Y; + goto finish; + } + else if (*substr == '-') { + ++substr; + --sublen; + } + else { + goto parse_error; + } + + /* Can't have a trailing '-' */ + if (sublen == 0) { + goto parse_error; + } + + /* PARSE THE MONTH (2 digits) */ + if (sublen >= 2 && isdigit(substr[0]) && isdigit(substr[1])) { + out->month = 10 * (substr[0] - '0') + (substr[1] - '0'); + + if (out->month < 1 || out->month > 12) { + PyErr_Format(PyExc_ValueError, + "Month out of range in datetime string \"%s\"", str); + goto error; + } + substr += 2; + sublen -= 2; + } + else { + goto parse_error; + } + + /* Next character must be a '-' or the end of the string */ + if (sublen == 0) { + if (out_local != NULL) { + *out_local = 0; + } + bestunit = NPY_FR_M; + goto finish; + } + else if (*substr == '-') { + ++substr; + --sublen; + } + else { + goto parse_error; + } + + /* Can't have a trailing '-' */ + if (sublen == 0) { + goto parse_error; + } + + /* PARSE THE DAY (2 digits) */ + if (sublen >= 2 && isdigit(substr[0]) && isdigit(substr[1])) { + out->day = 10 * (substr[0] - '0') + (substr[1] - '0'); + + if (out->day < 1 || + out->day > _days_per_month_table[year_leap][out->month-1]) { + PyErr_Format(PyExc_ValueError, + "Day out of range in datetime string \"%s\"", str); + goto error; + } + substr += 2; + sublen -= 2; + } + else { + goto parse_error; + } + + /* Next character must be a 'T', ' ', or end of string */ + if (sublen == 0) { + if (out_local != NULL) { + *out_local = 0; + } + bestunit = NPY_FR_D; + goto finish; + } + else if (*substr != 'T' && *substr != ' ') { + goto parse_error; + } + else { + ++substr; + --sublen; + } + + /* PARSE THE HOURS (2 digits) */ + if (sublen >= 2 && isdigit(substr[0]) && isdigit(substr[1])) { + out->hour = 10 * (substr[0] - '0') + (substr[1] - '0'); + + if (out->hour < 0 || out->hour >= 24) { + PyErr_Format(PyExc_ValueError, + "Hours out of range in datetime string \"%s\"", str); + goto error; + } + substr += 2; + sublen -= 2; + } + else { + goto parse_error; + } + + /* Next character must be a ':' or the end of the string */ + if (sublen > 0 && *substr == ':') { + ++substr; + --sublen; + } + else { + bestunit = NPY_FR_h; + goto parse_timezone; + } + + /* Can't have a trailing ':' */ + if (sublen == 0) { + goto parse_error; + } + + /* PARSE THE MINUTES (2 digits) */ + if (sublen >= 2 && isdigit(substr[0]) && isdigit(substr[1])) { + out->min = 10 * (substr[0] - '0') + (substr[1] - '0'); + + if (out->hour < 0 || out->min >= 60) { + PyErr_Format(PyExc_ValueError, + "Minutes out of range in datetime string \"%s\"", str); + goto error; + } + substr += 2; + sublen -= 2; + } + else { + goto parse_error; + } + + /* Next character must be a ':' or the end of the string */ + if (sublen > 0 && *substr == ':') { + ++substr; + --sublen; + } + else { + bestunit = NPY_FR_m; + goto parse_timezone; + } + + /* Can't have a trailing ':' */ + if (sublen == 0) { + goto parse_error; + } + + /* PARSE THE SECONDS (2 digits) */ + if (sublen >= 2 && isdigit(substr[0]) && isdigit(substr[1])) { + out->sec = 10 * (substr[0] - '0') + (substr[1] - '0'); + + if (out->sec < 0 || out->sec >= 60) { + PyErr_Format(PyExc_ValueError, + "Seconds out of range in datetime string \"%s\"", str); + goto error; + } + substr += 2; + sublen -= 2; + } + else { + goto parse_error; + } + + /* Next character may be a '.' indicating fractional seconds */ + if (sublen > 0 && *substr == '.') { + ++substr; + --sublen; + } + else { + bestunit = NPY_FR_s; + goto parse_timezone; + } + + /* PARSE THE MICROSECONDS (0 to 6 digits) */ + numdigits = 0; + for (i = 0; i < 6; ++i) { + out->us *= 10; + if (sublen > 0 && isdigit(*substr)) { + out->us += (*substr - '0'); + ++substr; + --sublen; + ++numdigits; + } + } + + if (sublen == 0 || !isdigit(*substr)) { + if (numdigits > 3) { + bestunit = NPY_FR_us; + } + else { + bestunit = NPY_FR_ms; + } + goto parse_timezone; + } + + /* PARSE THE PICOSECONDS (0 to 6 digits) */ + numdigits = 0; + for (i = 0; i < 6; ++i) { + out->ps *= 10; + if (sublen > 0 && isdigit(*substr)) { + out->ps += (*substr - '0'); + ++substr; + --sublen; + ++numdigits; + } + } + + if (sublen == 0 || !isdigit(*substr)) { + if (numdigits > 3) { + bestunit = NPY_FR_ps; + } + else { + bestunit = NPY_FR_ns; + } + goto parse_timezone; + } + + /* PARSE THE ATTOSECONDS (0 to 6 digits) */ + numdigits = 0; + for (i = 0; i < 6; ++i) { + out->as *= 10; + if (sublen > 0 && isdigit(*substr)) { + out->as += (*substr - '0'); + ++substr; + --sublen; + ++numdigits; + } + } + + if (numdigits > 3) { + bestunit = NPY_FR_as; + } + else { + bestunit = NPY_FR_fs; + } + +parse_timezone: + if (sublen == 0) { + /* + * ISO 8601 states to treat date-times without a timezone offset + * or 'Z' for UTC as local time. The C standard libary functions + * mktime and gmtime allow us to do this conversion. + * + * Only do this timezone adjustment for recent and future years. + */ + if (out->year > 1900 && out->year < 10000) { + time_t rawtime = 0; + struct tm tm_; + + tm_.tm_sec = out->sec; + tm_.tm_min = out->min; + tm_.tm_hour = out->hour; + tm_.tm_mday = out->day; + tm_.tm_mon = out->month - 1; + tm_.tm_year = out->year - 1900; + tm_.tm_isdst = -1; + + /* mktime converts a local 'struct tm' into a time_t */ + rawtime = mktime(&tm_); + if (rawtime == -1) { + PyErr_SetString(PyExc_OSError, "Failed to use mktime to " + "convert local time to UTC"); + goto error; + } + + /* gmtime converts a 'time_t' into a UTC 'struct tm' */ +#if defined(_WIN32) + if (gmtime_s(&tm_, &rawtime) != 0) { + PyErr_SetString(PyExc_OSError, "Failed to use gmtime_s to " + "get a UTC time"); + goto error; + } +#else + /* Other platforms may require something else */ + if (gmtime_r(&rawtime, &tm_) == NULL) { + PyErr_SetString(PyExc_OSError, "Failed to use gmtime_r to " + "get a UTC time"); + goto error; + } +#endif + out->sec = tm_.tm_sec; + out->min = tm_.tm_min; + out->hour = tm_.tm_hour; + out->day = tm_.tm_mday; + out->month = tm_.tm_mon + 1; + out->year = tm_.tm_year + 1900; + } + + /* Since neither "Z" nor a time-zone was specified, it's local */ + if (out_local != NULL) { + *out_local = 1; + } + + goto finish; + } + + /* UTC specifier */ + if (*substr == 'Z') { + /* "Z" means not local */ + if (out_local != NULL) { + *out_local = 0; + } + + if (sublen == 1) { + goto finish; + } + else { + ++substr; + --sublen; + } + } + /* Time zone offset */ + else if (*substr == '-' || *substr == '+') { + int offset_neg = 0, offset_hour = 0, offset_minute = 0; + + /* + * Since "local" means local with respect to the current + * machine, we say this is non-local. + */ + if (out_local != NULL) { + *out_local = 0; + } + + if (*substr == '-') { + offset_neg = 1; + } + ++substr; + --sublen; + + /* The hours offset */ + if (sublen >= 2 && isdigit(substr[0]) && isdigit(substr[1])) { + offset_hour = 10 * (substr[0] - '0') + (substr[1] - '0'); + substr += 2; + sublen -= 2; + if (offset_hour >= 24) { + PyErr_Format(PyExc_ValueError, + "Timezone hours offset out of range " + "in datetime string \"%s\"", str); + goto error; + } + } + else { + goto parse_error; + } + + /* The minutes offset is optional */ + if (sublen > 0) { + /* Optional ':' */ + if (*substr == ':') { + ++substr; + --sublen; + } + + /* The minutes offset (at the end of the string) */ + if (sublen >= 2 && isdigit(substr[0]) && isdigit(substr[1])) { + offset_minute = 10 * (substr[0] - '0') + (substr[1] - '0'); + substr += 2; + sublen -= 2; + if (offset_minute >= 60) { + PyErr_Format(PyExc_ValueError, + "Timezone minutes offset out of range " + "in datetime string \"%s\"", str); + goto error; + } + } + else { + goto parse_error; + } + } + + /* Apply the time zone offset */ + if (offset_neg) { + offset_hour = -offset_hour; + offset_minute = -offset_minute; + } + add_minutes_to_datetimestruct(out, -60 * offset_hour - offset_minute); + } + + /* Skip trailing whitespace */ + while (sublen > 0 && isspace(*substr)) { + ++substr; + --sublen; + } + + if (sublen != 0) { + goto parse_error; + } + +finish: + if (out_bestunit != NULL) { + *out_bestunit = bestunit; + } + + /* Check the casting rule */ + if (unit != -1 && !can_cast_datetime64_units(bestunit, unit, + casting)) { + PyErr_Format(PyExc_TypeError, "Cannot parse \"%s\" as unit " + "'%s' using casting rule %s", + str, _datetime_strings[unit], + npy_casting_to_string(casting)); + return -1; + } + + return 0; + +parse_error: + PyErr_Format(PyExc_ValueError, + "Error parsing datetime string \"%s\" at position %d", + str, (int)(substr-str)); + return -1; + +error: + return -1; +} + +/* + * Provides a string length to use for converting datetime + * objects with the given local and unit settings. + */ +NPY_NO_EXPORT int +get_datetime_iso_8601_strlen(int local, NPY_DATETIMEUNIT base) +{ + int len = 0; + + /* If no unit is provided, return the maximum length */ + if (base == -1) { + return NPY_DATETIME_MAX_ISO8601_STRLEN; + } + + switch (base) { + /* Generic units can only be used to represent NaT */ + case NPY_FR_GENERIC: + return 4; + case NPY_FR_as: + len += 3; /* "###" */ + case NPY_FR_fs: + len += 3; /* "###" */ + case NPY_FR_ps: + len += 3; /* "###" */ + case NPY_FR_ns: + len += 3; /* "###" */ + case NPY_FR_us: + len += 3; /* "###" */ + case NPY_FR_ms: + len += 4; /* ".###" */ + case NPY_FR_s: + len += 3; /* ":##" */ + case NPY_FR_m: + len += 3; /* ":##" */ + case NPY_FR_h: + len += 3; /* "T##" */ + case NPY_FR_D: + case NPY_FR_W: + len += 3; /* "-##" */ + case NPY_FR_M: + len += 3; /* "-##" */ + case NPY_FR_Y: + len += 21; /* 64-bit year */ + break; + } + + if (base >= NPY_FR_h) { + if (local) { + len += 5; /* "+####" or "-####" */ + } + else { + len += 1; /* "Z" */ + } + } + + len += 1; /* NULL terminator */ + + return len; +} + +/* + * Finds the largest unit whose value is nonzero, and for which + * the remainder for the rest of the units is zero. + */ +static NPY_DATETIMEUNIT +lossless_unit_from_datetimestruct(npy_datetimestruct *dts) +{ + if (dts->as % 1000 != 0) { + return NPY_FR_as; + } + else if (dts->as != 0) { + return NPY_FR_fs; + } + else if (dts->ps % 1000 != 0) { + return NPY_FR_ps; + } + else if (dts->ps != 0) { + return NPY_FR_ns; + } + else if (dts->us % 1000 != 0) { + return NPY_FR_us; + } + else if (dts->us != 0) { + return NPY_FR_ms; + } + else if (dts->sec != 0) { + return NPY_FR_s; + } + else if (dts->min != 0) { + return NPY_FR_m; + } + else if (dts->hour != 0) { + return NPY_FR_h; + } + else if (dts->day != 1) { + return NPY_FR_D; + } + else if (dts->month != 1) { + return NPY_FR_M; + } + else { + return NPY_FR_Y; + } +} + +/* + * Converts an npy_datetimestruct to an (almost) ISO 8601 + * NULL-terminated string. If the string fits in the space exactly, + * it leaves out the NULL terminator and returns success. + * + * The differences from ISO 8601 are the 'NaT' string, and + * the number of year digits is >= 4 instead of strictly 4. + * + * If 'local' is non-zero, it produces a string in local time with + * a +-#### timezone offset, otherwise it uses timezone Z (UTC). + * + * 'base' restricts the output to that unit. Set 'base' to + * -1 to auto-detect a base after which all the values are zero. + * + * 'tzoffset' is used if 'local' is enabled, and 'tzoffset' is + * set to a value other than -1. This is a manual override for + * the local time zone to use, as an offset in minutes. + * + * 'casting' controls whether data loss is allowed by truncating + * the data to a coarser unit. This interacts with 'local', slightly, + * in order to form a date unit string as a local time, the casting + * must be unsafe. + * + * Returns 0 on success, -1 on failure (for example if the output + * string was too short). + */ +NPY_NO_EXPORT int +make_iso_8601_datetime(npy_datetimestruct *dts, char *outstr, int outlen, + int local, NPY_DATETIMEUNIT base, int tzoffset, + NPY_CASTING casting) +{ + npy_datetimestruct dts_local; + int timezone_offset = 0; + + char *substr = outstr, sublen = outlen; + int tmplen; + + /* Handle NaT, and treat a datetime with generic units as NaT */ + if (dts->year == NPY_DATETIME_NAT || base == NPY_FR_GENERIC) { + if (outlen < 3) { + goto string_too_short; + } + outstr[0] = 'N'; + outstr[1] = 'a'; + outstr[2] = 'T'; + if (outlen > 3) { + outstr[3] = '\0'; + } + + return 0; + } + + /* Only do local time within a reasonable year range */ + if ((dts->year <= 1800 || dts->year >= 10000) && tzoffset == -1) { + local = 0; + } + + /* Automatically detect a good unit */ + if (base == -1) { + base = lossless_unit_from_datetimestruct(dts); + /* + * If there's a timezone, use at least minutes precision, + * and never split up hours and minutes by default + */ + if ((base < NPY_FR_m && local) || base == NPY_FR_h) { + base = NPY_FR_m; + } + /* Don't split up dates by default */ + else if (base < NPY_FR_D) { + base = NPY_FR_D; + } + } + /* + * Print weeks with the same precision as days. + * + * TODO: Could print weeks with YYYY-Www format if the week + * epoch is a Monday. + */ + else if (base == NPY_FR_W) { + base = NPY_FR_D; + } + + /* Use the C API to convert from UTC to local time */ + if (local && tzoffset == -1) { + time_t rawtime = 0, localrawtime; + struct tm tm_; + + /* + * Convert everything in 'dts' to a time_t, to minutes precision. + * This is POSIX time, which skips leap-seconds, but because + * we drop the seconds value from the npy_datetimestruct, everything + * is ok for this operation. + */ + rawtime = (time_t)get_datetimestruct_days(dts) * 24 * 60 * 60; + rawtime += dts->hour * 60 * 60; + rawtime += dts->min * 60; + + /* localtime converts a 'time_t' into a local 'struct tm' */ +#if defined(_WIN32) + if (localtime_s(&tm_, &rawtime) != 0) { + PyErr_SetString(PyExc_OSError, "Failed to use localtime_s to " + "get a local time"); + return -1; + } +#else + /* Other platforms may require something else */ + if (localtime_r(&rawtime, &tm_) == NULL) { + PyErr_SetString(PyExc_OSError, "Failed to use localtime_r to " + "get a local time"); + return -1; + } +#endif + /* Make a copy of the npy_datetimestruct we can modify */ + dts_local = *dts; + + /* Copy back all the values except seconds */ + dts_local.min = tm_.tm_min; + dts_local.hour = tm_.tm_hour; + dts_local.day = tm_.tm_mday; + dts_local.month = tm_.tm_mon + 1; + dts_local.year = tm_.tm_year + 1900; + + /* Extract the timezone offset that was applied */ + rawtime /= 60; + localrawtime = (time_t)get_datetimestruct_days(&dts_local) * 24 * 60; + localrawtime += dts_local.hour * 60; + localrawtime += dts_local.min; + + timezone_offset = localrawtime - rawtime; + + /* Set dts to point to our local time instead of the UTC time */ + dts = &dts_local; + } + /* Use the manually provided tzoffset */ + else if (local) { + /* Make a copy of the npy_datetimestruct we can modify */ + dts_local = *dts; + dts = &dts_local; + + /* Set and apply the required timezone offset */ + timezone_offset = tzoffset; + add_minutes_to_datetimestruct(dts, timezone_offset); + } + + /* + * Now the datetimestruct data is in the final form for + * the string representation, so ensure that the data + * is being cast according to the casting rule. + */ + if (casting != NPY_UNSAFE_CASTING) { + /* Producing a date as a local time is always 'unsafe' */ + if (base <= NPY_FR_D && local) { + PyErr_SetString(PyExc_TypeError, "Cannot create a local " + "timezone-based date string from a NumPy " + "datetime without forcing 'unsafe' casting"); + return -1; + } + /* Only 'unsafe' and 'same_kind' allow data loss */ + else { + NPY_DATETIMEUNIT unitprec; + + unitprec = lossless_unit_from_datetimestruct(dts); + if (casting != NPY_SAME_KIND_CASTING && unitprec > base) { + PyErr_Format(PyExc_TypeError, "Cannot create a " + "string with unit precision '%s' " + "from the NumPy datetime, which has data at " + "unit precision '%s', " + "requires 'unsafe' or 'same_kind' casting", + _datetime_strings[base], + _datetime_strings[unitprec]); + return -1; + } + } + } + + /* YEAR */ + /* + * Can't use PyOS_snprintf, because it always produces a '\0' + * character at the end, and NumPy string types are permitted + * to have data all the way to the end of the buffer. + */ +#ifdef _WIN32 + tmplen = _snprintf(substr, sublen, "%04" NPY_INT64_FMT, dts->year); +#else + tmplen = snprintf(substr, sublen, "%04" NPY_INT64_FMT, dts->year); +#endif + /* If it ran out of space or there isn't space for the NULL terminator */ + if (tmplen < 0 || tmplen > sublen) { + goto string_too_short; + } + substr += tmplen; + sublen -= tmplen; + + /* Stop if the unit is years */ + if (base == NPY_FR_Y) { + if (sublen > 0) { + *substr = '\0'; + } + return 0; + } + + /* MONTH */ + if (sublen < 1 ) { + goto string_too_short; + } + substr[0] = '-'; + if (sublen < 2 ) { + goto string_too_short; + } + substr[1] = (char)((dts->month / 10) + '0'); + if (sublen < 3 ) { + goto string_too_short; + } + substr[2] = (char)((dts->month % 10) + '0'); + substr += 3; + sublen -= 3; + + /* Stop if the unit is months */ + if (base == NPY_FR_M) { + if (sublen > 0) { + *substr = '\0'; + } + return 0; + } + + /* DAY */ + if (sublen < 1 ) { + goto string_too_short; + } + substr[0] = '-'; + if (sublen < 2 ) { + goto string_too_short; + } + substr[1] = (char)((dts->day / 10) + '0'); + if (sublen < 3 ) { + goto string_too_short; + } + substr[2] = (char)((dts->day % 10) + '0'); + substr += 3; + sublen -= 3; + + /* Stop if the unit is days */ + if (base == NPY_FR_D) { + if (sublen > 0) { + *substr = '\0'; + } + return 0; + } + + /* HOUR */ + if (sublen < 1 ) { + goto string_too_short; + } + substr[0] = 'T'; + if (sublen < 2 ) { + goto string_too_short; + } + substr[1] = (char)((dts->hour / 10) + '0'); + if (sublen < 3 ) { + goto string_too_short; + } + substr[2] = (char)((dts->hour % 10) + '0'); + substr += 3; + sublen -= 3; + + /* Stop if the unit is hours */ + if (base == NPY_FR_h) { + goto add_time_zone; + } + + /* MINUTE */ + if (sublen < 1 ) { + goto string_too_short; + } + substr[0] = ':'; + if (sublen < 2 ) { + goto string_too_short; + } + substr[1] = (char)((dts->min / 10) + '0'); + if (sublen < 3 ) { + goto string_too_short; + } + substr[2] = (char)((dts->min % 10) + '0'); + substr += 3; + sublen -= 3; + + /* Stop if the unit is minutes */ + if (base == NPY_FR_m) { + goto add_time_zone; + } + + /* SECOND */ + if (sublen < 1 ) { + goto string_too_short; + } + substr[0] = ':'; + if (sublen < 2 ) { + goto string_too_short; + } + substr[1] = (char)((dts->sec / 10) + '0'); + if (sublen < 3 ) { + goto string_too_short; + } + substr[2] = (char)((dts->sec % 10) + '0'); + substr += 3; + sublen -= 3; + + /* Stop if the unit is seconds */ + if (base == NPY_FR_s) { + goto add_time_zone; + } + + /* MILLISECOND */ + if (sublen < 1 ) { + goto string_too_short; + } + substr[0] = '.'; + if (sublen < 2 ) { + goto string_too_short; + } + substr[1] = (char)((dts->us / 100000) % 10 + '0'); + if (sublen < 3 ) { + goto string_too_short; + } + substr[2] = (char)((dts->us / 10000) % 10 + '0'); + if (sublen < 4 ) { + goto string_too_short; + } + substr[3] = (char)((dts->us / 1000) % 10 + '0'); + substr += 4; + sublen -= 4; + + /* Stop if the unit is milliseconds */ + if (base == NPY_FR_ms) { + goto add_time_zone; + } + + /* MICROSECOND */ + if (sublen < 1 ) { + goto string_too_short; + } + substr[0] = (char)((dts->us / 100) % 10 + '0'); + if (sublen < 2 ) { + goto string_too_short; + } + substr[1] = (char)((dts->us / 10) % 10 + '0'); + if (sublen < 3 ) { + goto string_too_short; + } + substr[2] = (char)(dts->us % 10 + '0'); + substr += 3; + sublen -= 3; + + /* Stop if the unit is microseconds */ + if (base == NPY_FR_us) { + goto add_time_zone; + } + + /* NANOSECOND */ + if (sublen < 1 ) { + goto string_too_short; + } + substr[0] = (char)((dts->ps / 100000) % 10 + '0'); + if (sublen < 2 ) { + goto string_too_short; + } + substr[1] = (char)((dts->ps / 10000) % 10 + '0'); + if (sublen < 3 ) { + goto string_too_short; + } + substr[2] = (char)((dts->ps / 1000) % 10 + '0'); + substr += 3; + sublen -= 3; + + /* Stop if the unit is nanoseconds */ + if (base == NPY_FR_ns) { + goto add_time_zone; + } + + /* PICOSECOND */ + if (sublen < 1 ) { + goto string_too_short; + } + substr[0] = (char)((dts->ps / 100) % 10 + '0'); + if (sublen < 2 ) { + goto string_too_short; + } + substr[1] = (char)((dts->ps / 10) % 10 + '0'); + if (sublen < 3 ) { + goto string_too_short; + } + substr[2] = (char)(dts->ps % 10 + '0'); + substr += 3; + sublen -= 3; + + /* Stop if the unit is picoseconds */ + if (base == NPY_FR_ps) { + goto add_time_zone; + } + + /* FEMTOSECOND */ + if (sublen < 1 ) { + goto string_too_short; + } + substr[0] = (char)((dts->as / 100000) % 10 + '0'); + if (sublen < 2 ) { + goto string_too_short; + } + substr[1] = (char)((dts->as / 10000) % 10 + '0'); + if (sublen < 3 ) { + goto string_too_short; + } + substr[2] = (char)((dts->as / 1000) % 10 + '0'); + substr += 3; + sublen -= 3; + + /* Stop if the unit is femtoseconds */ + if (base == NPY_FR_fs) { + goto add_time_zone; + } + + /* ATTOSECOND */ + if (sublen < 1 ) { + goto string_too_short; + } + substr[0] = (char)((dts->as / 100) % 10 + '0'); + if (sublen < 2 ) { + goto string_too_short; + } + substr[1] = (char)((dts->as / 10) % 10 + '0'); + if (sublen < 3 ) { + goto string_too_short; + } + substr[2] = (char)(dts->as % 10 + '0'); + substr += 3; + sublen -= 3; + +add_time_zone: + if (local) { + /* Add the +/- sign */ + if (sublen < 1) { + goto string_too_short; + } + if (timezone_offset < 0) { + substr[0] = '-'; + timezone_offset = -timezone_offset; + } + else { + substr[0] = '+'; + } + substr += 1; + sublen -= 1; + + /* Add the timezone offset */ + if (sublen < 1 ) { + goto string_too_short; + } + substr[0] = (char)((timezone_offset / (10*60)) % 10 + '0'); + if (sublen < 2 ) { + goto string_too_short; + } + substr[1] = (char)((timezone_offset / 60) % 10 + '0'); + if (sublen < 3 ) { + goto string_too_short; + } + substr[2] = (char)(((timezone_offset % 60) / 10) % 10 + '0'); + if (sublen < 4 ) { + goto string_too_short; + } + substr[3] = (char)((timezone_offset % 60) % 10 + '0'); + substr += 4; + sublen -= 4; + } + /* UTC "Zulu" time */ + else { + if (sublen < 1) { + goto string_too_short; + } + substr[0] = 'Z'; + substr += 1; + sublen -= 1; + } + + /* Add a NULL terminator, and return */ + if (sublen > 0) { + substr[0] = '\0'; + } + + return 0; + +string_too_short: + PyErr_Format(PyExc_RuntimeError, + "The string provided for NumPy ISO datetime formatting " + "was too short, with length %d", + outlen); + return -1; +} + + +/* + * This is the Python-exposed datetime_as_string function. + */ +NPY_NO_EXPORT PyObject * +array_datetime_as_string(PyObject *NPY_UNUSED(self), PyObject *args, + PyObject *kwds) +{ + PyObject *arr_in = NULL, *unit_in = NULL, *timezone = NULL; + NPY_DATETIMEUNIT unit; + NPY_CASTING casting = NPY_SAME_KIND_CASTING; + + int local = 0; + PyArray_DatetimeMetaData *meta; + int strsize; + + PyArrayObject *ret = NULL; + + NpyIter *iter = NULL; + PyArrayObject *op[2] = {NULL, NULL}; + PyArray_Descr *op_dtypes[2] = {NULL, NULL}; + npy_uint32 flags, op_flags[2]; + + static char *kwlist[] = {"arr", "unit", "timezone", "casting", NULL}; + + if(!PyArg_ParseTupleAndKeywords(args, kwds, + "O|OOO&:datetime_as_string", kwlist, + &arr_in, + &unit_in, + &timezone, + &PyArray_CastingConverter, &casting)) { + return NULL; + } + + /* Claim a reference to timezone for later */ + Py_XINCREF(timezone); + + op[0] = (PyArrayObject *)PyArray_FromAny(arr_in, + NULL, 0, 0, 0, NULL); + if (PyArray_DESCR(op[0])->type_num != NPY_DATETIME) { + PyErr_SetString(PyExc_TypeError, + "input must have type NumPy datetime"); + goto fail; + } + + /* Get the datetime metadata */ + meta = get_datetime_metadata_from_dtype(PyArray_DESCR(op[0])); + if (meta == NULL) { + goto fail; + } + + /* Use the metadata's unit for printing by default */ + unit = meta->base; + + /* Parse the input unit if provided */ + if (unit_in != NULL && unit_in != Py_None) { + PyObject *strobj; + char *str = NULL; + Py_ssize_t len = 0; + + if (PyUnicode_Check(unit_in)) { + strobj = PyUnicode_AsASCIIString(unit_in); + if (strobj == NULL) { + goto fail; + } + } + else { + strobj = unit_in; + Py_INCREF(strobj); + } + + if (PyBytes_AsStringAndSize(strobj, &str, &len) < 0) { + Py_DECREF(strobj); + goto fail; + } + + /* unit == -1 means to autodetect the unit from the datetime data */ + if (strcmp(str, "auto") == 0) { + unit = -1; + } + else { + unit = parse_datetime_unit_from_string(str, len, NULL); + if (unit == -1) { + Py_DECREF(strobj); + goto fail; + } + } + Py_DECREF(strobj); + + if (!can_cast_datetime64_units(meta->base, unit, casting)) { + PyErr_Format(PyExc_TypeError, "Cannot create a datetime " + "string as units '%s' from a NumPy datetime " + "with units '%s' according to the rule %s", + _datetime_strings[unit], + _datetime_strings[meta->base], + npy_casting_to_string(casting)); + goto fail; + } + } + + /* Get the input time zone */ + if (timezone != NULL) { + /* Convert to ASCII if it's unicode */ + if (PyUnicode_Check(timezone)) { + /* accept unicode input */ + PyObject *obj_str; + obj_str = PyUnicode_AsASCIIString(timezone); + if (obj_str == NULL) { + goto fail; + } + Py_DECREF(timezone); + timezone = obj_str; + } + + /* Check for the supported string inputs */ + if (PyBytes_Check(timezone)) { + char *str; + Py_ssize_t len; + + if (PyBytes_AsStringAndSize(timezone, &str, &len) < 0) { + goto fail; + } + + if (strcmp(str, "local") == 0) { + local = 1; + Py_DECREF(timezone); + timezone = NULL; + } + else if (strcmp(str, "UTC") == 0) { + local = 0; + Py_DECREF(timezone); + timezone = NULL; + } + else { + PyErr_Format(PyExc_ValueError, "Unsupported timezone " + "input string \"%s\"", str); + goto fail; + } + } + /* Otherwise assume it's a Python TZInfo, or acts like one */ + else { + local = 1; + } + } + + /* Create the output string data type with a big enough length */ + op_dtypes[1] = PyArray_DescrNewFromType(NPY_STRING); + if (op_dtypes[1] == NULL) { + goto fail; + } + strsize = get_datetime_iso_8601_strlen(local, unit); + op_dtypes[1]->elsize = strsize; + + flags = NPY_ITER_ZEROSIZE_OK| + NPY_ITER_BUFFERED; + op_flags[0] = NPY_ITER_READONLY| + NPY_ITER_ALIGNED; + op_flags[1] = NPY_ITER_WRITEONLY| + NPY_ITER_ALLOCATE; + + iter = NpyIter_MultiNew(2, op, flags, NPY_KEEPORDER, NPY_NO_CASTING, + op_flags, op_dtypes); + if (iter == NULL) { + goto fail; + } + + if (NpyIter_GetIterSize(iter) != 0) { + NpyIter_IterNextFunc *iternext; + char **dataptr; + npy_datetime dt; + npy_datetimestruct dts; + + iternext = NpyIter_GetIterNext(iter, NULL); + if (iternext == NULL) { + goto fail; + } + dataptr = NpyIter_GetDataPtrArray(iter); + + do { + int tzoffset = -1; + + /* Get the datetime */ + dt = *(npy_datetime *)dataptr[0]; + + /* Convert it to a struct */ + if (convert_datetime_to_datetimestruct(meta, dt, &dts) < 0) { + goto fail; + } + + /* Get the tzoffset from the timezone if provided */ + if (local && timezone != NULL) { + tzoffset = get_tzoffset_from_pytzinfo(timezone, &dts); + if (tzoffset == -1) { + goto fail; + } + } + + /* Zero the destination string completely */ + memset(dataptr[1], 0, strsize); + /* Convert that into a string */ + if (make_iso_8601_datetime(&dts, (char *)dataptr[1], strsize, + local, unit, tzoffset, casting) < 0) { + goto fail; + } + } while(iternext(iter)); + } + + ret = NpyIter_GetOperandArray(iter)[1]; + Py_INCREF(ret); + + Py_XDECREF(timezone); + Py_XDECREF(op[0]); + Py_XDECREF(op[1]); + Py_XDECREF(op_dtypes[0]); + Py_XDECREF(op_dtypes[1]); + if (iter != NULL) { + NpyIter_Deallocate(iter); + } + + return PyArray_Return(ret); + +fail: + Py_XDECREF(timezone); + Py_XDECREF(op[0]); + Py_XDECREF(op[1]); + Py_XDECREF(op_dtypes[0]); + Py_XDECREF(op_dtypes[1]); + if (iter != NULL) { + NpyIter_Deallocate(iter); + } + + return NULL; +} + + + diff --git a/numpy/core/src/multiarray/datetime_strings.h b/numpy/core/src/multiarray/datetime_strings.h new file mode 100644 index 000000000000..f27856b89d78 --- /dev/null +++ b/numpy/core/src/multiarray/datetime_strings.h @@ -0,0 +1,89 @@ +#ifndef _NPY_PRIVATE__DATETIME_STRINGS_H_ +#define _NPY_PRIVATE__DATETIME_STRINGS_H_ + +/* + * Parses (almost) standard ISO 8601 date strings. The differences are: + * + * + The date "20100312" is parsed as the year 20100312, not as + * equivalent to "2010-03-12". The '-' in the dates are not optional. + * + Only seconds may have a decimal point, with up to 18 digits after it + * (maximum attoseconds precision). + * + Either a 'T' as in ISO 8601 or a ' ' may be used to separate + * the date and the time. Both are treated equivalently. + * + Doesn't (yet) handle the "YYYY-DDD" or "YYYY-Www" formats. + * + Doesn't handle leap seconds (seconds value has 60 in these cases). + * + Doesn't handle 24:00:00 as synonym for midnight (00:00:00) tomorrow + * + Accepts special values "NaT" (not a time), "Today", (current + * day according to local time) and "Now" (current time in UTC). + * + * 'str' must be a NULL-terminated string, and 'len' must be its length. + * 'unit' should contain -1 if the unit is unknown, or the unit + * which will be used if it is. + * 'casting' controls how the detected unit from the string is allowed + * to be cast to the 'unit' parameter. + * + * 'out' gets filled with the parsed date-time. + * 'out_local' gets set to 1 if the parsed time was in local time, + * to 0 otherwise. The values 'now' and 'today' don't get counted + * as local, and neither do UTC +/-#### timezone offsets, because + * they aren't using the computer's local timezone offset. + * 'out_bestunit' gives a suggested unit based on the amount of + * resolution provided in the string, or -1 for NaT. + * 'out_special' gets set to 1 if the parsed time was 'today', + * 'now', or ''/'NaT'. For 'today', the unit recommended is + * 'D', for 'now', the unit recommended is 's', and for 'NaT' + * the unit recommended is 'Y'. + * + * Returns 0 on success, -1 on failure. + */ +NPY_NO_EXPORT int +parse_iso_8601_datetime(char *str, int len, + NPY_DATETIMEUNIT unit, + NPY_CASTING casting, + npy_datetimestruct *out, + npy_bool *out_local, + NPY_DATETIMEUNIT *out_bestunit, + npy_bool *out_special); + +/* + * Provides a string length to use for converting datetime + * objects with the given local and unit settings. + */ +NPY_NO_EXPORT int +get_datetime_iso_8601_strlen(int local, NPY_DATETIMEUNIT base); + +/* + * Converts an npy_datetimestruct to an (almost) ISO 8601 + * NULL-terminated string. + * + * If 'local' is non-zero, it produces a string in local time with + * a +-#### timezone offset, otherwise it uses timezone Z (UTC). + * + * 'base' restricts the output to that unit. Set 'base' to + * -1 to auto-detect a base after which all the values are zero. + * + * 'tzoffset' is used if 'local' is enabled, and 'tzoffset' is + * set to a value other than -1. This is a manual override for + * the local time zone to use, as an offset in minutes. + * + * 'casting' controls whether data loss is allowed by truncating + * the data to a coarser unit. This interacts with 'local', slightly, + * in order to form a date unit string as a local time, the casting + * must be unsafe. + * + * Returns 0 on success, -1 on failure (for example if the output + * string was too short). + */ +NPY_NO_EXPORT int +make_iso_8601_datetime(npy_datetimestruct *dts, char *outstr, int outlen, + int local, NPY_DATETIMEUNIT base, int tzoffset, + NPY_CASTING casting); + +/* + * This is the Python-exposed datetime_as_string function. + */ +NPY_NO_EXPORT PyObject * +array_datetime_as_string(PyObject *NPY_UNUSED(self), PyObject *args, + PyObject *kwds); + +#endif diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index 6016917115c4..2e793fe1f652 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -1793,7 +1793,7 @@ arraydescr_new(PyTypeObject *NPY_UNUSED(subtype), PyObject *args, PyObject *kwds /* * Return a tuple of - * (cleaned metadata dictionary, tuple with (str, num, events)) + * (cleaned metadata dictionary, tuple with (str, num)) */ static PyObject * _get_pickleabletype_from_datetime_metadata(PyArray_Descr *dtype) @@ -1904,7 +1904,7 @@ arraydescr_reduce(PyArray_Descr *self, PyObject *NPY_UNUSED(args)) /* Handle CObject in NPY_METADATA_DTSTR key separately */ /* * newobj is a tuple of cleaned metadata dictionary - * and tuple of date_time info (str, num, den, events) + * and tuple of date_time info (str, num) */ newobj = _get_pickleabletype_from_datetime_metadata(self); if (newobj == NULL) { diff --git a/numpy/core/src/multiarray/dtype_transfer.c b/numpy/core/src/multiarray/dtype_transfer.c index fa5573ad54f6..445133ebf299 100644 --- a/numpy/core/src/multiarray/dtype_transfer.c +++ b/numpy/core/src/multiarray/dtype_transfer.c @@ -20,7 +20,9 @@ #include "numpy/npy_3kcompat.h" +#include "convert_datatype.h" #include "_datetime.h" +#include "datetime_strings.h" #include "lowlevel_strided_loops.h" @@ -330,7 +332,8 @@ _strided_to_strided_contig_align_wrap(char *dst, npy_intp dst_stride, PyArray_StridedTransferFn *wrapped = d->wrapped, *tobuffer = d->tobuffer, *frombuffer = d->frombuffer; - npy_intp dst_itemsize = d->dst_itemsize; + npy_intp inner_src_itemsize = d->src_itemsize, + dst_itemsize = d->dst_itemsize; void *wrappeddata = d->wrappeddata, *todata = d->todata, *fromdata = d->fromdata; @@ -338,12 +341,12 @@ _strided_to_strided_contig_align_wrap(char *dst, npy_intp dst_stride, for(;;) { if (N > NPY_LOWLEVEL_BUFFER_BLOCKSIZE) { - tobuffer(bufferin, src_itemsize, src, src_stride, + tobuffer(bufferin, inner_src_itemsize, src, src_stride, NPY_LOWLEVEL_BUFFER_BLOCKSIZE, src_itemsize, todata); - wrapped(bufferout, dst_itemsize, bufferin, src_itemsize, + wrapped(bufferout, dst_itemsize, bufferin, inner_src_itemsize, NPY_LOWLEVEL_BUFFER_BLOCKSIZE, - src_itemsize, wrappeddata); + inner_src_itemsize, wrappeddata); frombuffer(dst, dst_stride, bufferout, dst_itemsize, NPY_LOWLEVEL_BUFFER_BLOCKSIZE, dst_itemsize, fromdata); @@ -352,10 +355,10 @@ _strided_to_strided_contig_align_wrap(char *dst, npy_intp dst_stride, dst += NPY_LOWLEVEL_BUFFER_BLOCKSIZE*dst_stride; } else { - tobuffer(bufferin, src_itemsize, src, src_stride, N, + tobuffer(bufferin, inner_src_itemsize, src, src_stride, N, src_itemsize, todata); - wrapped(bufferout, dst_itemsize, bufferin, src_itemsize, N, - src_itemsize, wrappeddata); + wrapped(bufferout, dst_itemsize, bufferin, inner_src_itemsize, N, + inner_src_itemsize, wrappeddata); frombuffer(dst, dst_stride, bufferout, dst_itemsize, N, dst_itemsize, fromdata); return; @@ -373,7 +376,8 @@ _strided_to_strided_contig_align_wrap_init_dest(char *dst, npy_intp dst_stride, PyArray_StridedTransferFn *wrapped = d->wrapped, *tobuffer = d->tobuffer, *frombuffer = d->frombuffer; - npy_intp dst_itemsize = d->dst_itemsize; + npy_intp inner_src_itemsize = d->src_itemsize, + dst_itemsize = d->dst_itemsize; void *wrappeddata = d->wrappeddata, *todata = d->todata, *fromdata = d->fromdata; @@ -381,13 +385,13 @@ _strided_to_strided_contig_align_wrap_init_dest(char *dst, npy_intp dst_stride, for(;;) { if (N > NPY_LOWLEVEL_BUFFER_BLOCKSIZE) { - tobuffer(bufferin, src_itemsize, src, src_stride, + tobuffer(bufferin, inner_src_itemsize, src, src_stride, NPY_LOWLEVEL_BUFFER_BLOCKSIZE, src_itemsize, todata); memset(bufferout, 0, dst_itemsize*NPY_LOWLEVEL_BUFFER_BLOCKSIZE); - wrapped(bufferout, dst_itemsize, bufferin, src_itemsize, + wrapped(bufferout, dst_itemsize, bufferin, inner_src_itemsize, NPY_LOWLEVEL_BUFFER_BLOCKSIZE, - src_itemsize, wrappeddata); + inner_src_itemsize, wrappeddata); frombuffer(dst, dst_stride, bufferout, dst_itemsize, NPY_LOWLEVEL_BUFFER_BLOCKSIZE, dst_itemsize, fromdata); @@ -396,11 +400,11 @@ _strided_to_strided_contig_align_wrap_init_dest(char *dst, npy_intp dst_stride, dst += NPY_LOWLEVEL_BUFFER_BLOCKSIZE*dst_stride; } else { - tobuffer(bufferin, src_itemsize, src, src_stride, N, + tobuffer(bufferin, inner_src_itemsize, src, src_stride, N, src_itemsize, todata); memset(bufferout, 0, dst_itemsize*N); - wrapped(bufferout, dst_itemsize, bufferin, src_itemsize, N, - src_itemsize, wrappeddata); + wrapped(bufferout, dst_itemsize, bufferin, inner_src_itemsize, N, + inner_src_itemsize, wrappeddata); frombuffer(dst, dst_stride, bufferout, dst_itemsize, N, dst_itemsize, fromdata); return; @@ -699,14 +703,23 @@ get_nbo_cast_numeric_transfer_function(int aligned, return NPY_SUCCEED; } -/* Does a datetime->datetime or timedelta->timedelta cast */ +/* + * Does a datetime->datetime, timedelta->timedelta, + * datetime->ascii, or ascii->datetime cast + */ typedef struct { free_strided_transfer_data freefunc; copy_strided_transfer_data copyfunc; /* The conversion fraction */ npy_int64 num, denom; - /* The number of events in the source and destination */ - int src_events, dst_events; + /* For the datetime -> string conversion, the dst string length */ + npy_intp src_itemsize, dst_itemsize; + /* + * A buffer of size 'src_itemsize + 1', for when the input + * string is exactly of length src_itemsize with no NULL + * terminator. + */ + char *tmp_buffer; /* * The metadata for when dealing with Months or Years * which behave non-linearly with respect to the other @@ -715,7 +728,17 @@ typedef struct { PyArray_DatetimeMetaData src_meta, dst_meta; } _strided_datetime_cast_data; -/* strided cast data copy function */ +/* strided datetime cast data free function */ +void _strided_datetime_cast_data_free(void *data) +{ + _strided_datetime_cast_data *d = (_strided_datetime_cast_data *)data; + if (d->tmp_buffer != NULL) { + PyArray_free(d->tmp_buffer); + } + PyArray_free(data); +} + +/* strided datetime cast data copy function */ void *_strided_datetime_cast_data_copy(void *data) { _strided_datetime_cast_data *newdata = @@ -726,6 +749,13 @@ void *_strided_datetime_cast_data_copy(void *data) } memcpy(newdata, data, sizeof(_strided_datetime_cast_data)); + if (newdata->tmp_buffer != NULL) { + newdata->tmp_buffer = PyArray_malloc(newdata->src_itemsize + 1); + if (newdata->tmp_buffer == NULL) { + PyArray_free(newdata); + return NULL; + } + } return (void *)newdata; } @@ -748,7 +778,6 @@ _strided_to_strided_datetime_general_cast(char *dst, npy_intp dst_stride, dt = NPY_DATETIME_NAT; } else { - dts.event = dts.event % d->dst_meta.events; if (convert_datetimestruct_to_datetime(&d->dst_meta, &dts, &dt) < 0) { dt = NPY_DATETIME_NAT; @@ -772,22 +801,11 @@ _strided_to_strided_datetime_cast(char *dst, npy_intp dst_stride, _strided_datetime_cast_data *d = (_strided_datetime_cast_data *)data; npy_int64 num = d->num, denom = d->denom; npy_int64 dt; - int event = 0, src_events = d->src_events, dst_events = d->dst_events; while (N > 0) { memcpy(&dt, src, sizeof(dt)); if (dt != NPY_DATETIME_NAT) { - /* Remove the event number from the value */ - if (src_events > 1) { - event = (int)(dt % src_events); - dt = dt / src_events; - if (event < 0) { - --dt; - event += src_events; - } - } - /* Apply the scaling */ if (dt < 0) { dt = (dt * num - (denom - 1)) / denom; @@ -795,12 +813,6 @@ _strided_to_strided_datetime_cast(char *dst, npy_intp dst_stride, else { dt = dt * num / denom; } - - /* Add the event number back in */ - if (dst_events > 1) { - event = event % dst_events; - dt = dt * dst_events + event; - } } memcpy(dst, &dt, sizeof(dt)); @@ -812,7 +824,7 @@ _strided_to_strided_datetime_cast(char *dst, npy_intp dst_stride, } static void -_aligned_strided_to_strided_datetime_cast_no_events(char *dst, +_aligned_strided_to_strided_datetime_cast(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp src_itemsize, @@ -843,6 +855,94 @@ _aligned_strided_to_strided_datetime_cast_no_events(char *dst, } } +static void +_strided_to_strided_datetime_to_string(char *dst, npy_intp dst_stride, + char *src, npy_intp src_stride, + npy_intp N, npy_intp NPY_UNUSED(src_itemsize), + void *data) +{ + _strided_datetime_cast_data *d = (_strided_datetime_cast_data *)data; + npy_intp dst_itemsize = d->dst_itemsize; + npy_int64 dt; + npy_datetimestruct dts; + + while (N > 0) { + memcpy(&dt, src, sizeof(dt)); + + if (convert_datetime_to_datetimestruct(&d->src_meta, + dt, &dts) < 0) { + /* For an error, produce a 'NaT' string */ + dts.year = NPY_DATETIME_NAT; + } + + /* Initialize the destination to all zeros */ + memset(dst, 0, dst_itemsize); + + /* + * This may also raise an error, but the caller needs + * to use PyErr_Occurred(). + */ + make_iso_8601_datetime(&dts, dst, dst_itemsize, + 0, d->src_meta.base, -1, + NPY_UNSAFE_CASTING); + + dst += dst_stride; + src += src_stride; + --N; + } +} + +static void +_strided_to_strided_string_to_datetime(char *dst, npy_intp dst_stride, + char *src, npy_intp src_stride, + npy_intp N, npy_intp src_itemsize, + void *data) +{ + _strided_datetime_cast_data *d = (_strided_datetime_cast_data *)data; + npy_int64 dt; + npy_datetimestruct dts; + char *tmp_buffer = d->tmp_buffer; + char *tmp; + + while (N > 0) { + /* Replicating strnlen with memchr, because Mac OS X lacks it */ + tmp = memchr(src, '\0', src_itemsize); + + /* If the string is all full, use the buffer */ + if (tmp == NULL) { + memcpy(tmp_buffer, src, src_itemsize); + tmp_buffer[src_itemsize] = '\0'; + + if (parse_iso_8601_datetime(tmp_buffer, src_itemsize, + d->dst_meta.base, NPY_SAME_KIND_CASTING, + &dts, NULL, NULL, NULL) < 0) { + dt = NPY_DATETIME_NAT; + } + } + /* Otherwise parse the data in place */ + else { + if (parse_iso_8601_datetime(src, tmp - src, + d->dst_meta.base, NPY_SAME_KIND_CASTING, + &dts, NULL, NULL, NULL) < 0) { + dt = NPY_DATETIME_NAT; + } + } + + /* Convert to the datetime */ + if (dt != NPY_DATETIME_NAT && + convert_datetimestruct_to_datetime(&d->dst_meta, + &dts, &dt) < 0) { + dt = NPY_DATETIME_NAT; + } + + memcpy(dst, &dt, sizeof(dt)); + + dst += dst_stride; + src += src_stride; + --N; + } +} + /* * Assumes src_dtype and dst_dtype are both datetimes or both timedeltas */ @@ -881,12 +981,11 @@ get_nbo_cast_datetime_transfer_function(int aligned, *out_transferdata = NULL; return NPY_FAIL; } - data->freefunc = &PyArray_free; + data->freefunc = &_strided_datetime_cast_data_free; data->copyfunc = &_strided_datetime_cast_data_copy; data->num = num; data->denom = denom; - data->src_events = src_meta->events; - data->dst_events = dst_meta->events; + data->tmp_buffer = NULL; /* * Special case the datetime (but not timedelta) with the nonlinear @@ -902,8 +1001,8 @@ get_nbo_cast_datetime_transfer_function(int aligned, memcpy(&data->dst_meta, dst_meta, sizeof(data->dst_meta)); *out_stransfer = &_strided_to_strided_datetime_general_cast; } - else if (aligned && data->src_events == 1 && data->dst_events == 1) { - *out_stransfer = &_aligned_strided_to_strided_datetime_cast_no_events; + else if (aligned) { + *out_stransfer = &_aligned_strided_to_strided_datetime_cast; } else { *out_stransfer = &_strided_to_strided_datetime_cast; @@ -923,6 +1022,244 @@ get_nbo_cast_datetime_transfer_function(int aligned, return NPY_SUCCEED; } +static int +get_nbo_datetime_to_string_transfer_function(int aligned, + npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + PyArray_StridedTransferFn **out_stransfer, + void **out_transferdata) +{ + PyArray_DatetimeMetaData *src_meta; + _strided_datetime_cast_data *data; + + src_meta = get_datetime_metadata_from_dtype(src_dtype); + if (src_meta == NULL) { + return NPY_FAIL; + } + + /* Allocate the data for the casting */ + data = (_strided_datetime_cast_data *)PyArray_malloc( + sizeof(_strided_datetime_cast_data)); + if (data == NULL) { + PyErr_NoMemory(); + *out_stransfer = NULL; + *out_transferdata = NULL; + return NPY_FAIL; + } + data->freefunc = &_strided_datetime_cast_data_free; + data->copyfunc = &_strided_datetime_cast_data_copy; + data->dst_itemsize = dst_dtype->elsize; + data->tmp_buffer = NULL; + + memcpy(&data->src_meta, src_meta, sizeof(data->src_meta)); + + *out_stransfer = &_strided_to_strided_datetime_to_string; + *out_transferdata = data; + +#if NPY_DT_DBG_TRACING + printf("Dtype transfer from "); + PyObject_Print((PyObject *)src_dtype, stdout, 0); + printf(" to "); + PyObject_Print((PyObject *)dst_dtype, stdout, 0); + printf("\n"); +#endif + + return NPY_SUCCEED; +} + +static int +get_datetime_to_unicode_transfer_function(int aligned, + npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + PyArray_StridedTransferFn **out_stransfer, + void **out_transferdata, + int *out_needs_api) +{ + void *castdata = NULL, *todata = NULL, *fromdata = NULL; + PyArray_StridedTransferFn *caststransfer, *tobuffer, *frombuffer; + PyArray_Descr *str_dtype; + + /* Get an ASCII string data type, adapted to match the UNICODE one */ + str_dtype = PyArray_DescrFromType(NPY_STRING); + PyArray_AdaptFlexibleDType(NULL, dst_dtype, &str_dtype); + if (str_dtype == NULL) { + return NPY_FAIL; + } + + /* Get the copy/swap operation to dst */ + if (PyArray_GetDTypeCopySwapFn(aligned, + src_stride, src_dtype->elsize, + src_dtype, + &tobuffer, &todata) != NPY_SUCCEED) { + Py_DECREF(str_dtype); + return NPY_FAIL; + } + + /* Get the NBO datetime to string aligned contig function */ + if (get_nbo_datetime_to_string_transfer_function(1, + src_dtype->elsize, str_dtype->elsize, + src_dtype, str_dtype, + &caststransfer, &castdata) != NPY_SUCCEED) { + Py_DECREF(str_dtype); + PyArray_FreeStridedTransferData(todata); + return NPY_FAIL; + } + + /* Get the cast operation to dst */ + if (PyArray_GetDTypeTransferFunction(aligned, + str_dtype->elsize, dst_stride, + str_dtype, dst_dtype, + 0, + &frombuffer, &fromdata, + out_needs_api) != NPY_SUCCEED) { + Py_DECREF(str_dtype); + PyArray_FreeStridedTransferData(todata); + PyArray_FreeStridedTransferData(castdata); + return NPY_FAIL; + } + + /* Wrap it all up in a new transfer function + data */ + if (wrap_aligned_contig_transfer_function( + src_dtype->elsize, str_dtype->elsize, + tobuffer, todata, + frombuffer, fromdata, + caststransfer, castdata, + PyDataType_FLAGCHK(str_dtype, NPY_NEEDS_INIT), + out_stransfer, out_transferdata) != NPY_SUCCEED) { + PyArray_FreeStridedTransferData(castdata); + PyArray_FreeStridedTransferData(todata); + PyArray_FreeStridedTransferData(fromdata); + return NPY_FAIL; + } + + Py_DECREF(str_dtype); + + return NPY_SUCCEED; +} + +static int +get_nbo_string_to_datetime_transfer_function(int aligned, + npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + PyArray_StridedTransferFn **out_stransfer, + void **out_transferdata) +{ + PyArray_DatetimeMetaData *dst_meta; + _strided_datetime_cast_data *data; + + dst_meta = get_datetime_metadata_from_dtype(dst_dtype); + if (dst_meta == NULL) { + return NPY_FAIL; + } + + /* Allocate the data for the casting */ + data = (_strided_datetime_cast_data *)PyArray_malloc( + sizeof(_strided_datetime_cast_data)); + if (data == NULL) { + PyErr_NoMemory(); + *out_stransfer = NULL; + *out_transferdata = NULL; + return NPY_FAIL; + } + data->freefunc = &_strided_datetime_cast_data_free; + data->copyfunc = &_strided_datetime_cast_data_copy; + data->src_itemsize = src_dtype->elsize; + data->tmp_buffer = PyArray_malloc(data->src_itemsize + 1); + if (data->tmp_buffer == NULL) { + PyErr_NoMemory(); + PyArray_free(data); + *out_stransfer = NULL; + *out_transferdata = NULL; + return NPY_FAIL; + } + + memcpy(&data->dst_meta, dst_meta, sizeof(data->dst_meta)); + + *out_stransfer = &_strided_to_strided_string_to_datetime; + *out_transferdata = data; + +#if NPY_DT_DBG_TRACING + printf("Dtype transfer from "); + PyObject_Print((PyObject *)src_dtype, stdout, 0); + printf(" to "); + PyObject_Print((PyObject *)dst_dtype, stdout, 0); + printf("\n"); +#endif + + return NPY_SUCCEED; +} + +static int +get_unicode_to_datetime_transfer_function(int aligned, + npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + PyArray_StridedTransferFn **out_stransfer, + void **out_transferdata, + int *out_needs_api) +{ + void *castdata = NULL, *todata = NULL, *fromdata = NULL; + PyArray_StridedTransferFn *caststransfer, *tobuffer, *frombuffer; + PyArray_Descr *str_dtype; + + /* Get an ASCII string data type, adapted to match the UNICODE one */ + str_dtype = PyArray_DescrFromType(NPY_STRING); + PyArray_AdaptFlexibleDType(NULL, src_dtype, &str_dtype); + if (str_dtype == NULL) { + return NPY_FAIL; + } + + /* Get the cast operation from src */ + if (PyArray_GetDTypeTransferFunction(aligned, + src_stride, str_dtype->elsize, + src_dtype, str_dtype, + 0, + &tobuffer, &todata, + out_needs_api) != NPY_SUCCEED) { + Py_DECREF(str_dtype); + return NPY_FAIL; + } + + /* Get the string to NBO datetime aligned contig function */ + if (get_nbo_string_to_datetime_transfer_function(1, + str_dtype->elsize, dst_dtype->elsize, + str_dtype, dst_dtype, + &caststransfer, &castdata) != NPY_SUCCEED) { + Py_DECREF(str_dtype); + PyArray_FreeStridedTransferData(todata); + return NPY_FAIL; + } + + /* Get the copy/swap operation to dst */ + if (PyArray_GetDTypeCopySwapFn(aligned, + dst_dtype->elsize, dst_stride, + dst_dtype, + &frombuffer, &fromdata) != NPY_SUCCEED) { + Py_DECREF(str_dtype); + PyArray_FreeStridedTransferData(todata); + PyArray_FreeStridedTransferData(castdata); + return NPY_FAIL; + } + + /* Wrap it all up in a new transfer function + data */ + if (wrap_aligned_contig_transfer_function( + str_dtype->elsize, dst_dtype->elsize, + tobuffer, todata, + frombuffer, fromdata, + caststransfer, castdata, + PyDataType_FLAGCHK(dst_dtype, NPY_NEEDS_INIT), + out_stransfer, out_transferdata) != NPY_SUCCEED) { + Py_DECREF(str_dtype); + PyArray_FreeStridedTransferData(castdata); + PyArray_FreeStridedTransferData(todata); + PyArray_FreeStridedTransferData(fromdata); + return NPY_FAIL; + } + + Py_DECREF(str_dtype); + + return NPY_SUCCEED; +} + static int get_nbo_cast_transfer_function(int aligned, npy_intp src_stride, npy_intp dst_stride, @@ -949,17 +1286,68 @@ get_nbo_cast_transfer_function(int aligned, out_stransfer, out_transferdata); } - /* As a parameterized type, datetime->datetime sometimes needs casting */ - if ((src_dtype->type_num == NPY_DATETIME && - dst_dtype->type_num == NPY_DATETIME) || - (src_dtype->type_num == NPY_TIMEDELTA && - dst_dtype->type_num == NPY_TIMEDELTA)) { - *out_needs_wrap = !PyArray_ISNBO(src_dtype->byteorder) || - !PyArray_ISNBO(dst_dtype->byteorder); - return get_nbo_cast_datetime_transfer_function(aligned, - src_stride, dst_stride, - src_dtype, dst_dtype, - out_stransfer, out_transferdata); + if (src_dtype->type_num == NPY_DATETIME || + src_dtype->type_num == NPY_TIMEDELTA || + dst_dtype->type_num == NPY_DATETIME || + dst_dtype->type_num == NPY_TIMEDELTA) { + /* A parameterized type, datetime->datetime sometimes needs casting */ + if ((src_dtype->type_num == NPY_DATETIME && + dst_dtype->type_num == NPY_DATETIME) || + (src_dtype->type_num == NPY_TIMEDELTA && + dst_dtype->type_num == NPY_TIMEDELTA)) { + *out_needs_wrap = !PyArray_ISNBO(src_dtype->byteorder) || + !PyArray_ISNBO(dst_dtype->byteorder); + return get_nbo_cast_datetime_transfer_function(aligned, + src_stride, dst_stride, + src_dtype, dst_dtype, + out_stransfer, out_transferdata); + } + + /* + * Datetime <-> string conversions can be handled specially. + * The functions may raise an error if the strings have no + * space, or can't be parsed properly. + */ + if (src_dtype->type_num == NPY_DATETIME) { + switch (dst_dtype->type_num) { + case NPY_STRING: + *out_needs_api = 1; + *out_needs_wrap = !PyArray_ISNBO(src_dtype->byteorder); + return get_nbo_datetime_to_string_transfer_function( + aligned, + src_stride, dst_stride, + src_dtype, dst_dtype, + out_stransfer, out_transferdata); + + case NPY_UNICODE: + return get_datetime_to_unicode_transfer_function( + aligned, + src_stride, dst_stride, + src_dtype, dst_dtype, + out_stransfer, out_transferdata, + out_needs_api); + } + } + else if (dst_dtype->type_num == NPY_DATETIME) { + switch (src_dtype->type_num) { + case NPY_STRING: + *out_needs_api = 1; + *out_needs_wrap = !PyArray_ISNBO(dst_dtype->byteorder); + return get_nbo_string_to_datetime_transfer_function( + aligned, + src_stride, dst_stride, + src_dtype, dst_dtype, + out_stransfer, out_transferdata); + + case NPY_UNICODE: + return get_unicode_to_datetime_transfer_function( + aligned, + src_stride, dst_stride, + src_dtype, dst_dtype, + out_stransfer, out_transferdata, + out_needs_api); + } + } } *out_needs_wrap = !aligned || @@ -1128,69 +1516,17 @@ get_cast_transfer_function(int aligned, PyArray_StridedTransferFn *tobuffer, *frombuffer; /* Get the copy/swap operation from src */ - - /* If it's a custom data type, wrap its copy swap function */ - if (src_dtype->type_num >= NPY_NTYPES) { - tobuffer = NULL; - wrap_copy_swap_function(aligned, + PyArray_GetDTypeCopySwapFn(aligned, src_stride, src_itemsize, src_dtype, - !PyArray_ISNBO(src_dtype->byteorder), &tobuffer, &todata); - } - /* A straight copy */ - else if (src_itemsize == 1 || PyArray_ISNBO(src_dtype->byteorder)) { - tobuffer = PyArray_GetStridedCopyFn(aligned, - src_stride, src_itemsize, - src_itemsize); - } - /* If it's not complex, one swap */ - else if(src_dtype->kind != 'c') { - tobuffer = PyArray_GetStridedCopySwapFn(aligned, - src_stride, src_itemsize, - src_itemsize); - } - /* If complex, a paired swap */ - else { - tobuffer = PyArray_GetStridedCopySwapPairFn(aligned, - src_stride, src_itemsize, - src_itemsize); - } - /* Get the copy/swap operation to dst */ - /* If it's a custom data type, wrap its copy swap function */ - if (dst_dtype->type_num >= NPY_NTYPES) { - frombuffer = NULL; - wrap_copy_swap_function(aligned, + /* Get the copy/swap operation to dst */ + PyArray_GetDTypeCopySwapFn(aligned, dst_itemsize, dst_stride, dst_dtype, - !PyArray_ISNBO(dst_dtype->byteorder), &frombuffer, &fromdata); - } - /* A straight copy */ - else if (dst_itemsize == 1 || PyArray_ISNBO(dst_dtype->byteorder)) { - if (dst_dtype->type_num == NPY_OBJECT) { - frombuffer = &_strided_to_strided_move_references; - } - else { - frombuffer = PyArray_GetStridedCopyFn(aligned, - dst_itemsize, dst_stride, - dst_itemsize); - } - } - /* If it's not complex, one swap */ - else if(dst_dtype->kind != 'c') { - frombuffer = PyArray_GetStridedCopySwapFn(aligned, - dst_itemsize, dst_stride, - dst_itemsize); - } - /* If complex, a paired swap */ - else { - frombuffer = PyArray_GetStridedCopySwapPairFn(aligned, - dst_itemsize, dst_stride, - dst_itemsize); - } if (frombuffer == NULL || tobuffer == NULL) { PyArray_FreeStridedTransferData(castdata); @@ -3014,6 +3350,51 @@ get_decsrcref_transfer_function(int aligned, } } +/********************* DTYPE COPY SWAP FUNCTION ***********************/ + +NPY_NO_EXPORT int +PyArray_GetDTypeCopySwapFn(int aligned, + npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *dtype, + PyArray_StridedTransferFn **outstransfer, + void **outtransferdata) +{ + npy_intp itemsize = dtype->elsize; + + /* If it's a custom data type, wrap its copy swap function */ + if (dtype->type_num >= NPY_NTYPES) { + *outstransfer = NULL; + wrap_copy_swap_function(aligned, + src_stride, dst_stride, + dtype, + !PyArray_ISNBO(dtype->byteorder), + outstransfer, outtransferdata); + } + /* A straight copy */ + else if (itemsize == 1 || PyArray_ISNBO(dtype->byteorder)) { + *outstransfer = PyArray_GetStridedCopyFn(aligned, + src_stride, dst_stride, + itemsize); + *outtransferdata = NULL; + } + /* If it's not complex, one swap */ + else if(dtype->kind != 'c') { + *outstransfer = PyArray_GetStridedCopySwapFn(aligned, + src_stride, dst_stride, + itemsize); + *outtransferdata = NULL; + } + /* If complex, a paired swap */ + else { + *outstransfer = PyArray_GetStridedCopySwapPairFn(aligned, + src_stride, dst_stride, + itemsize); + *outtransferdata = NULL; + } + + return (*outstransfer == NULL) ? NPY_FAIL : NPY_SUCCEED; +} + /********************* MAIN DTYPE TRANSFER FUNCTION ***********************/ NPY_NO_EXPORT int @@ -3072,6 +3453,7 @@ PyArray_GetDTypeTransferFunction(int aligned, PyTypeNum_ISNUMBER(dst_type_num) && PyArray_ISNBO(src_dtype->byteorder) && PyArray_ISNBO(dst_dtype->byteorder)) { + if (PyArray_EquivTypenums(src_type_num, dst_type_num)) { *out_stransfer = PyArray_GetStridedCopyFn(aligned, src_stride, dst_stride, diff --git a/numpy/core/src/multiarray/lowlevel_strided_loops.c.src b/numpy/core/src/multiarray/lowlevel_strided_loops.c.src index 0a1786bc419d..170a7167e835 100644 --- a/numpy/core/src/multiarray/lowlevel_strided_loops.c.src +++ b/numpy/core/src/multiarray/lowlevel_strided_loops.c.src @@ -311,7 +311,7 @@ _contig_to_contig(char *dst, npy_intp NPY_UNUSED(dst_stride), NPY_NO_EXPORT PyArray_StridedTransferFn * -PyArray_GetStridedCopyFn(npy_intp aligned, npy_intp src_stride, +PyArray_GetStridedCopyFn(int aligned, npy_intp src_stride, npy_intp dst_stride, npy_intp itemsize) { /* @@ -466,7 +466,7 @@ PyArray_GetStridedCopyFn(npy_intp aligned, npy_intp src_stride, */ NPY_NO_EXPORT PyArray_StridedTransferFn * -@function@(npy_intp aligned, npy_intp src_stride, +@function@(int aligned, npy_intp src_stride, npy_intp dst_stride, npy_intp itemsize) { /* @@ -848,7 +848,7 @@ static void /**end repeat**/ NPY_NO_EXPORT PyArray_StridedTransferFn * -PyArray_GetStridedNumericCastFn(npy_intp aligned, npy_intp src_stride, +PyArray_GetStridedNumericCastFn(int aligned, npy_intp src_stride, npy_intp dst_stride, int src_type_num, int dst_type_num) { diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index 9502bc9f04ad..2cd4487acd87 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -17,6 +17,7 @@ #include "calculation.h" #include "methods.h" +#include "convert_datatype.h" /* NpyArg_ParseKeywords @@ -766,7 +767,7 @@ array_setasflat(PyArrayObject *self, PyObject *args) Py_RETURN_NONE; } -static const char * +NPY_NO_EXPORT const char * npy_casting_to_string(NPY_CASTING casting) { switch (casting) { @@ -831,33 +832,20 @@ array_astype(PyArrayObject *self, PyObject *args, PyObject *kwds) else if (PyArray_CanCastArrayTo(self, dtype, casting)) { PyArrayObject *ret; - if (dtype->elsize == 0) { - PyArray_DESCR_REPLACE(dtype); - if (dtype == NULL) { - return NULL; - } - - if (dtype->type_num == PyArray_DESCR(self)->type_num || - dtype->type_num == NPY_VOID) { - dtype->elsize = PyArray_DESCR(self)->elsize; - } - else if (PyArray_DESCR(self)->type_num == NPY_STRING && - dtype->type_num == NPY_UNICODE) { - dtype->elsize = PyArray_DESCR(self)->elsize * 4; - } - else if (PyArray_DESCR(self)->type_num == NPY_UNICODE && - dtype->type_num == NPY_STRING) { - dtype->elsize = PyArray_DESCR(self)->elsize / 4; - } + /* If the requested dtype is flexible, adapt it */ + PyArray_AdaptFlexibleDType((PyObject *)self, PyArray_DESCR(self), + &dtype); + if (dtype == NULL) { + return NULL; } - + /* This steals the reference to dtype, so no DECREF of dtype */ ret = (PyArrayObject *)PyArray_NewLikeArray( self, order, dtype, subok); - if (ret == NULL) { return NULL; } + if (PyArray_CopyInto(ret, self) < 0) { Py_DECREF(ret); return NULL; diff --git a/numpy/core/src/multiarray/methods.h b/numpy/core/src/multiarray/methods.h index 642265ccdc60..fc3b987fad90 100644 --- a/numpy/core/src/multiarray/methods.h +++ b/numpy/core/src/multiarray/methods.h @@ -5,4 +5,7 @@ extern NPY_NO_EXPORT PyMethodDef array_methods[]; #endif +NPY_NO_EXPORT const char * +npy_casting_to_string(NPY_CASTING casting); + #endif diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 3b136e072c00..3b537a5563bf 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -45,6 +45,7 @@ NPY_NO_EXPORT int NPY_NUMUSERTYPES = 0; #include "convert_datatype.h" #include "nditer_pywrap.h" #include "_datetime.h" +#include "datetime_strings.h" #include "datetime_busday.h" #include "datetime_busdaycal.h" @@ -2808,170 +2809,6 @@ array_datetime_data(PyObject *NPY_UNUSED(dummy), PyObject *args) return convert_datetime_metadata_to_tuple(meta); } -static PyObject * -array_datetime_as_string(PyObject *NPY_UNUSED(self), PyObject *args, - PyObject *kwds) -{ - PyObject *arr_in = NULL, *unit_in = NULL; - int local = 0, tzoffset = -1; - NPY_DATETIMEUNIT unit; - PyArray_DatetimeMetaData *meta; - int strsize; - - PyArrayObject *ret = NULL; - - NpyIter *iter = NULL; - PyArrayObject *op[2] = {NULL, NULL}; - PyArray_Descr *op_dtypes[2] = {NULL, NULL}; - npy_uint32 flags, op_flags[2]; - - static char *kwlist[] = {"arr", "local", "unit", "tzoffset", NULL}; - - if(!PyArg_ParseTupleAndKeywords(args, kwds, - "O|iOi:datetime_as_string", kwlist, - &arr_in, - &local, - &unit_in, - &tzoffset)) { - goto fail; - } - - op[0] = (PyArrayObject *)PyArray_FromAny(arr_in, - NULL, 0, 0, 0, NULL); - if (PyArray_DESCR(op[0])->type_num != NPY_DATETIME) { - PyErr_SetString(PyExc_TypeError, - "input must have type NumPy datetime"); - goto fail; - } - - /* Get the datetime metadata */ - meta = get_datetime_metadata_from_dtype(PyArray_DESCR(op[0])); - if (meta == NULL) { - goto fail; - } - - /* Use the metadata's unit for printing by default */ - unit = meta->base; - - /* Parse the input unit if provided */ - if (unit_in != NULL && unit_in != Py_None) { - PyObject *strobj; - char *str = NULL; - Py_ssize_t len = 0; - - if (PyUnicode_Check(unit_in)) { - strobj = PyUnicode_AsASCIIString(unit_in); - if (strobj == NULL) { - goto fail; - } - } - else { - strobj = unit_in; - Py_INCREF(strobj); - } - - if (PyBytes_AsStringAndSize(strobj, &str, &len) < 0) { - Py_DECREF(strobj); - goto fail; - } - - /* unit == -1 means to autodetect the unit from the datetime data */ - if (strcmp(str, "auto") == 0) { - unit = -1; - } - else { - unit = parse_datetime_unit_from_string(str, len, NULL); - if (unit == -1) { - Py_DECREF(strobj); - goto fail; - } - } - Py_DECREF(strobj); - } - - if (!local && tzoffset != -1) { - PyErr_SetString(PyExc_ValueError, - "Can only use 'tzoffset' parameter when 'local' is " - "set to True"); - goto fail; - } - - /* Create the output string data type with a big enough length */ - op_dtypes[1] = PyArray_DescrNewFromType(NPY_STRING); - if (op_dtypes[1] == NULL) { - goto fail; - } - strsize = get_datetime_iso_8601_strlen(local, unit); - op_dtypes[1]->elsize = strsize; - - flags = NPY_ITER_ZEROSIZE_OK| - NPY_ITER_BUFFERED; - op_flags[0] = NPY_ITER_READONLY| - NPY_ITER_ALIGNED; - op_flags[1] = NPY_ITER_WRITEONLY| - NPY_ITER_ALLOCATE; - - iter = NpyIter_MultiNew(2, op, flags, NPY_KEEPORDER, NPY_NO_CASTING, - op_flags, op_dtypes); - if (iter == NULL) { - goto fail; - } - - if (NpyIter_GetIterSize(iter) != 0) { - NpyIter_IterNextFunc *iternext; - char **dataptr; - npy_datetime dt; - npy_datetimestruct dts; - - iternext = NpyIter_GetIterNext(iter, NULL); - if (iternext == NULL) { - goto fail; - } - dataptr = NpyIter_GetDataPtrArray(iter); - - do { - /* Get the datetime */ - dt = *(datetime *)dataptr[0]; - /* Convert it to a struct */ - if (convert_datetime_to_datetimestruct(meta, dt, &dts) < 0) { - goto fail; - } - /* Zero the destination string completely */ - memset(dataptr[1], 0, strsize); - /* Convert that into a string */ - if (make_iso_8601_date(&dts, (char *)dataptr[1], strsize, - local, unit, tzoffset) < 0) { - goto fail; - } - } while(iternext(iter)); - } - - ret = NpyIter_GetOperandArray(iter)[1]; - Py_INCREF(ret); - - Py_XDECREF(op[0]); - Py_XDECREF(op[1]); - Py_XDECREF(op_dtypes[0]); - Py_XDECREF(op_dtypes[1]); - if (iter != NULL) { - NpyIter_Deallocate(iter); - } - - return PyArray_Return(ret); - -fail: - Py_XDECREF(op[0]); - Py_XDECREF(op[1]); - Py_XDECREF(op_dtypes[0]); - Py_XDECREF(op_dtypes[1]); - if (iter != NULL) { - NpyIter_Deallocate(iter); - } - - return NULL; -} - - #if !defined(NPY_PY3K) static PyObject * new_buffer(PyObject *NPY_UNUSED(dummy), PyObject *args) diff --git a/numpy/core/src/multiarray/multiarraymodule_onefile.c b/numpy/core/src/multiarray/multiarraymodule_onefile.c index 4459e6b4ccc2..bcfe73e0f3ce 100644 --- a/numpy/core/src/multiarray/multiarraymodule_onefile.c +++ b/numpy/core/src/multiarray/multiarraymodule_onefile.c @@ -11,6 +11,7 @@ #include "scalarapi.c" #include "datetime.c" +#include "datetime_strings.c" #include "datetime_busday.c" #include "datetime_busdaycal.c" #include "arraytypes.c" diff --git a/numpy/core/src/multiarray/nditer.c.src b/numpy/core/src/multiarray/nditer.c.src index 3d7be4137d43..dfc0baa6f70c 100644 --- a/numpy/core/src/multiarray/nditer.c.src +++ b/numpy/core/src/multiarray/nditer.c.src @@ -3145,37 +3145,13 @@ npyiter_prepare_one_operand(PyArrayObject **op, if (op_request_dtype != NULL) { /* We just have a borrowed reference to op_request_dtype */ Py_INCREF(op_request_dtype); - /* If it's a data type without a size, set the size */ - if (op_request_dtype->elsize == 0) { - PyArray_DESCR_REPLACE(op_request_dtype); - if (op_request_dtype == NULL) { - return 0; - } - - if (op_request_dtype->type_num == NPY_STRING) { - switch((*op_dtype)->type_num) { - case NPY_STRING: - op_request_dtype->elsize = (*op_dtype)->elsize; - break; - case NPY_UNICODE: - op_request_dtype->elsize = (*op_dtype)->elsize >> 2; - break; - } - } - else if (op_request_dtype->type_num == NPY_UNICODE) { - switch((*op_dtype)->type_num) { - case NPY_STRING: - op_request_dtype->elsize = (*op_dtype)->elsize << 2; - break; - case NPY_UNICODE: - op_request_dtype->elsize = (*op_dtype)->elsize; - break; - } - } - else if (op_request_dtype->type_num == NPY_VOID) { - op_request_dtype->elsize = (*op_dtype)->elsize; - } + /* If the requested dtype is flexible, adapt it */ + PyArray_AdaptFlexibleDType((PyObject *)(*op), PyArray_DESCR(*op), + &op_request_dtype); + if (op_request_dtype == NULL) { + return 0; } + /* Store the requested dtype */ Py_DECREF(*op_dtype); *op_dtype = op_request_dtype; diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index 380ec3493d7d..0e7eb5914089 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -22,6 +22,7 @@ #include "common.h" #include "scalartypes.h" #include "_datetime.h" +#include "datetime_strings.h" NPY_NO_EXPORT PyBoolScalarObject _PyArrayScalar_BoolValues[] = { {PyObject_HEAD_INIT(&PyBoolArrType_Type) 0}, @@ -604,6 +605,8 @@ datetimetype_repr(PyObject *self) npy_datetimestruct dts; PyObject *ret; char iso[NPY_DATETIME_MAX_ISO8601_STRLEN]; + int local; + NPY_DATETIMEUNIT unit; if (!PyArray_IsScalar(self, Datetime)) { PyErr_SetString(PyExc_RuntimeError, @@ -618,19 +621,42 @@ datetimetype_repr(PyObject *self) return NULL; } - if (make_iso_8601_date(&dts, iso, sizeof(iso), 1, - scal->obmeta.base, -1) < 0) { + local = (scal->obmeta.base > NPY_FR_D); + /* + * Because we're defaulting to local time, display hours with + * minutes precision, so that 30-minute timezone offsets can work. + */ + unit = scal->obmeta.base; + if (unit == NPY_FR_h) { + unit = NPY_FR_m; + } + if (make_iso_8601_datetime(&dts, iso, sizeof(iso), local, + unit, -1, NPY_SAFE_CASTING) < 0) { return NULL; } - ret = PyUString_FromString("numpy.datetime64('"); - PyUString_ConcatAndDel(&ret, - PyUString_FromString(iso)); - PyUString_ConcatAndDel(&ret, - PyUString_FromString("','")); - ret = append_metastr_to_string(&scal->obmeta, 1, ret); - PyUString_ConcatAndDel(&ret, - PyUString_FromString("')")); + /* + * For straight units or generic units, the unit will be deduced + * from the string, so it's not necessary to specify it. + */ + if ((scal->obmeta.num == 1 && scal->obmeta.base != NPY_FR_h) || + scal->obmeta.base == NPY_FR_GENERIC) { + ret = PyUString_FromString("numpy.datetime64('"); + PyUString_ConcatAndDel(&ret, + PyUString_FromString(iso)); + PyUString_ConcatAndDel(&ret, + PyUString_FromString("')")); + } + else { + ret = PyUString_FromString("numpy.datetime64('"); + PyUString_ConcatAndDel(&ret, + PyUString_FromString(iso)); + PyUString_ConcatAndDel(&ret, + PyUString_FromString("','")); + ret = append_metastr_to_string(&scal->obmeta, 1, ret); + PyUString_ConcatAndDel(&ret, + PyUString_FromString("')")); + } return ret; } @@ -649,12 +675,25 @@ timedeltatype_repr(PyObject *self) scal = (PyTimedeltaScalarObject *)self; - ret = PyUString_FromFormat("numpy.timedelta64(%lld", scal->obval); - PyUString_ConcatAndDel(&ret, - PyUString_FromString(",'")); - ret = append_metastr_to_string(&scal->obmeta, 1, ret); - PyUString_ConcatAndDel(&ret, - PyUString_FromString("')")); + /* The value */ + if (scal->obval == NPY_DATETIME_NAT) { + ret = PyUString_FromString("numpy.timedelta64('NaT'"); + } + else { + ret = PyUString_FromFormat("numpy.timedelta64(%lld", scal->obval); + } + /* The metadata unit */ + if (scal->obmeta.base == NPY_FR_GENERIC) { + PyUString_ConcatAndDel(&ret, + PyUString_FromString(")")); + } + else { + PyUString_ConcatAndDel(&ret, + PyUString_FromString(",'")); + ret = append_metastr_to_string(&scal->obmeta, 1, ret); + PyUString_ConcatAndDel(&ret, + PyUString_FromString("')")); + } return ret; } @@ -665,6 +704,8 @@ datetimetype_str(PyObject *self) PyDatetimeScalarObject *scal; npy_datetimestruct dts; char iso[NPY_DATETIME_MAX_ISO8601_STRLEN]; + int local; + NPY_DATETIMEUNIT unit; if (!PyArray_IsScalar(self, Datetime)) { PyErr_SetString(PyExc_RuntimeError, @@ -679,15 +720,24 @@ datetimetype_str(PyObject *self) return NULL; } - if (make_iso_8601_date(&dts, iso, sizeof(iso), 1, - scal->obmeta.base, -1) < 0) { + local = (scal->obmeta.base > NPY_FR_D); + /* + * Because we're defaulting to local time, display hours with + * minutes precision, so that 30-minute timezone offsets can work. + */ + unit = scal->obmeta.base; + if (unit == NPY_FR_h) { + unit = NPY_FR_m; + } + if (make_iso_8601_datetime(&dts, iso, sizeof(iso), local, + unit, -1, NPY_SAFE_CASTING) < 0) { return NULL; } return PyUString_FromString(iso); } -static char *_datetime_verbose_strings[] = { +static char *_datetime_verbose_strings[NPY_DATETIME_NUMUNITS] = { "years", "months", "weeks", @@ -719,8 +769,6 @@ timedeltatype_str(PyObject *self) scal = (PyTimedeltaScalarObject *)self; - /* TODO: Account for events, etc */ - if (scal->obmeta.base >= 0 && scal->obmeta.base < NPY_DATETIME_NUMUNITS) { basestr = _datetime_verbose_strings[scal->obmeta.base]; } @@ -730,10 +778,15 @@ timedeltatype_str(PyObject *self) return NULL; } - ret = PyUString_FromFormat("%lld ", + if (scal->obval == NPY_DATETIME_NAT) { + ret = PyUString_FromString("NaT"); + } + else { + ret = PyUString_FromFormat("%lld ", (long long)(scal->obval * scal->obmeta.num)); - PyUString_ConcatAndDel(&ret, - PyUString_FromString(basestr)); + PyUString_ConcatAndDel(&ret, + PyUString_FromString(basestr)); + } return ret; } @@ -2533,7 +2586,6 @@ static PyObject * if (ret->obmeta.base == -1) { ret->obmeta.base = NPY_DATETIME_DEFAULTUNIT; ret->obmeta.num = 1; - ret->obmeta.events = 1; } /* Make datetime default to NaT, timedelta default to zero */ @@ -2543,8 +2595,8 @@ static PyObject * ret->obval = 0; #endif } - else if (convert_pyobject_to_@name@(&ret->obmeta, obj, &ret->obval) - < 0) { + else if (convert_pyobject_to_@name@(&ret->obmeta, obj, + NPY_SAME_KIND_CASTING, &ret->obval) < 0) { Py_DECREF(ret); return NULL; } diff --git a/numpy/core/src/private/lowlevel_strided_loops.h b/numpy/core/src/private/lowlevel_strided_loops.h index 79b8f6fd7a20..166ece29a515 100644 --- a/numpy/core/src/private/lowlevel_strided_loops.h +++ b/numpy/core/src/private/lowlevel_strided_loops.h @@ -60,8 +60,9 @@ PyArray_CopyStridedTransferData(void *transferdata); * */ NPY_NO_EXPORT PyArray_StridedTransferFn * -PyArray_GetStridedCopyFn(npy_intp aligned, npy_intp src_stride, - npy_intp dst_stride, npy_intp itemsize); +PyArray_GetStridedCopyFn(int aligned, + npy_intp src_stride, npy_intp dst_stride, + npy_intp itemsize); /* * Gives back a function pointer to a specialized function for copying @@ -74,8 +75,9 @@ PyArray_GetStridedCopyFn(npy_intp aligned, npy_intp src_stride, * Parameters are as for PyArray_GetStridedCopyFn. */ NPY_NO_EXPORT PyArray_StridedTransferFn * -PyArray_GetStridedCopySwapFn(npy_intp aligned, npy_intp src_stride, - npy_intp dst_stride, npy_intp itemsize); +PyArray_GetStridedCopySwapFn(int aligned, + npy_intp src_stride, npy_intp dst_stride, + npy_intp itemsize); /* * Gives back a function pointer to a specialized function for copying @@ -88,8 +90,9 @@ PyArray_GetStridedCopySwapFn(npy_intp aligned, npy_intp src_stride, * Parameters are as for PyArray_GetStridedCopyFn. */ NPY_NO_EXPORT PyArray_StridedTransferFn * -PyArray_GetStridedCopySwapPairFn(npy_intp aligned, npy_intp src_stride, - npy_intp dst_stride, npy_intp itemsize); +PyArray_GetStridedCopySwapPairFn(int aligned, + npy_intp src_stride, npy_intp dst_stride, + npy_intp itemsize); /* * Gives back a transfer function and transfer data pair which copies @@ -115,9 +118,22 @@ PyArray_GetStridedZeroPadCopyFn(int aligned, * without setting a Python exception. */ NPY_NO_EXPORT PyArray_StridedTransferFn * -PyArray_GetStridedNumericCastFn(npy_intp aligned, npy_intp src_stride, - npy_intp dst_stride, - int src_type_num, int dst_type_num); +PyArray_GetStridedNumericCastFn(int aligned, + npy_intp src_stride, npy_intp dst_stride, + int src_type_num, int dst_type_num); + +/* + * Gets an operation which copies elements of the given dtype, + * swapping if the dtype isn't in NBO. + * + * Returns NPY_SUCCEED or NPY_FAIL + */ +NPY_NO_EXPORT int +PyArray_GetDTypeCopySwapFn(int aligned, + npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *dtype, + PyArray_StridedTransferFn **outstransfer, + void **outtransferdata); /* * If it's possible, gives back a transfer function which casts and/or diff --git a/numpy/core/src/umath/umathmodule.c.src b/numpy/core/src/umath/umathmodule.c.src index b4cece358b5d..a6da7c2dde36 100644 --- a/numpy/core/src/umath/umathmodule.c.src +++ b/numpy/core/src/umath/umathmodule.c.src @@ -43,7 +43,8 @@ static PyUFuncGenericFunction pyfunc_functions[] = {PyUFunc_On_Om}; -static int object_ufunc_type_resolution(PyUFuncObject *ufunc, +static int +object_ufunc_type_resolution(PyUFuncObject *ufunc, NPY_CASTING casting, PyArrayObject **operands, PyObject *type_tup, diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index c533f3b2c2e3..f975923709bb 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -77,11 +77,18 @@ def test_datetime_casting_rules(self): # Cannot cast timedelta safely from months/years to days assert_(not np.can_cast('m8[M]', 'm8[D]', casting='safe')) assert_(not np.can_cast('m8[Y]', 'm8[D]', casting='safe')) - # Can cast same_kind from months/years to days + # Can cast datetime same_kind from months/years to days assert_(np.can_cast('M8[M]', 'M8[D]', casting='same_kind')) assert_(np.can_cast('M8[Y]', 'M8[D]', casting='same_kind')) - assert_(np.can_cast('m8[M]', 'm8[D]', casting='same_kind')) - assert_(np.can_cast('m8[Y]', 'm8[D]', casting='same_kind')) + # Can't cast timedelta same_kind from months/years to days + assert_(not np.can_cast('m8[M]', 'm8[D]', casting='same_kind')) + assert_(not np.can_cast('m8[Y]', 'm8[D]', casting='same_kind')) + # Can't cast datetime same_kind across the date/time boundary + assert_(not np.can_cast('M8[D]', 'M8[h]', casting='same_kind')) + assert_(not np.can_cast('M8[h]', 'M8[D]', casting='same_kind')) + # Can cast timedelta same_kind across the date/time boundary + assert_(np.can_cast('m8[D]', 'm8[h]', casting='same_kind')) + assert_(np.can_cast('m8[h]', 'm8[D]', casting='same_kind')) # Cannot cast safely if the integer multiplier doesn't divide assert_(not np.can_cast('M8[7h]', 'M8[3h]', casting='safe')) @@ -95,12 +102,23 @@ def test_datetime_scalar_construction(self): # Construct with different units assert_equal(np.datetime64('1950-03-12', 'D'), np.datetime64('1950-03-12')) - assert_equal(np.datetime64('1950-03-12', 's'), - np.datetime64('1950-03-12', 'm')) + assert_equal(np.datetime64('1950-03-12T13Z', 's'), + np.datetime64('1950-03-12T13Z', 'm')) # Default construction means NaT assert_equal(np.datetime64(), np.datetime64('NaT')) + # Some basic strings and repr + assert_equal(str(np.datetime64('NaT')), 'NaT') + assert_equal(repr(np.datetime64('NaT')), + "numpy.datetime64('NaT')") + assert_equal(str(np.datetime64('2011-02')), '2011-02') + assert_equal(repr(np.datetime64('2011-02')), + "numpy.datetime64('2011-02')") + + # None gets constructed as NaT + assert_equal(np.datetime64(None), np.datetime64('NaT')) + # Default construction of NaT is in generic units assert_equal(np.datetime64().dtype, np.dtype('M8')) assert_equal(np.datetime64('NaT').dtype, np.dtype('M8')) @@ -138,6 +156,19 @@ def test_datetime_scalar_construction(self): np.datetime64(datetime.datetime(1980,1,25, 14,36,22,500000))) + # Construction with time units from a date raises + assert_raises(TypeError, np.datetime64, '1920-03-13', 'h') + assert_raises(TypeError, np.datetime64, '1920-03', 'm') + assert_raises(TypeError, np.datetime64, '1920', 's') + assert_raises(TypeError, np.datetime64, datetime.date(2045,3,25), 'ms') + # Construction with date units from a datetime raises + assert_raises(TypeError, np.datetime64, '1920-03-13T18Z', 'D') + assert_raises(TypeError, np.datetime64, '1920-03-13T18:33Z', 'W') + assert_raises(TypeError, np.datetime64, '1920-03-13T18:33:12Z', 'M') + assert_raises(TypeError, np.datetime64, '1920-03-13T18:33:12.5Z', 'Y') + assert_raises(TypeError, np.datetime64, + datetime.datetime(1920,4,14,13,20), 'D') + def test_timedelta_scalar_construction(self): # Construct with different units assert_equal(np.timedelta64(7, 'D'), @@ -148,6 +179,19 @@ def test_timedelta_scalar_construction(self): # Default construction means 0 assert_equal(np.timedelta64(), np.timedelta64(0)) + # None gets constructed as NaT + assert_equal(np.timedelta64(None), np.timedelta64('NaT')) + + # Some basic strings and repr + assert_equal(str(np.timedelta64('NaT')), 'NaT') + assert_equal(repr(np.timedelta64('NaT')), + "numpy.timedelta64('NaT')") + assert_equal(str(np.timedelta64(3, 's')), '3 seconds') + assert_equal(repr(np.timedelta64(-3, 's')), + "numpy.timedelta64(-3,'s')") + assert_equal(repr(np.timedelta64(12)), + "numpy.timedelta64(12)") + # Construction from an integer produces generic units assert_equal(np.timedelta64(12).dtype, np.dtype('m8')) @@ -191,6 +235,17 @@ def test_timedelta_scalar_construction(self): assert_equal(np.timedelta64(28, 'W'), np.timedelta64(datetime.timedelta(weeks=28))) + # Cannot construct across nonlinear time unit boundaries + a = np.timedelta64(3, 's') + assert_raises(TypeError, np.timedelta64, a, 'M') + assert_raises(TypeError, np.timedelta64, a, 'Y') + a = np.timedelta64(6, 'M') + assert_raises(TypeError, np.timedelta64, a, 'D') + assert_raises(TypeError, np.timedelta64, a, 'h') + a = np.timedelta64(1, 'Y') + assert_raises(TypeError, np.timedelta64, a, 'D') + assert_raises(TypeError, np.timedelta64, a, 'm') + def test_timedelta_scalar_construction_units(self): # String construction detecting units assert_equal(np.datetime64('2010').dtype, @@ -267,17 +322,17 @@ def test_timedelta_scalar_construction_units(self): assert_equal(np.datetime64('today').dtype, np.dtype('M8[D]')) - assert_raises(ValueError, np.datetime64, 'today', 'h') - assert_raises(ValueError, np.datetime64, 'today', 's') - assert_raises(ValueError, np.datetime64, 'today', 'as') + assert_raises(TypeError, np.datetime64, 'today', 'h') + assert_raises(TypeError, np.datetime64, 'today', 's') + assert_raises(TypeError, np.datetime64, 'today', 'as') # 'now' special value assert_equal(np.datetime64('now').dtype, np.dtype('M8[s]')) - assert_raises(ValueError, np.datetime64, 'now', 'Y') - assert_raises(ValueError, np.datetime64, 'now', 'M') - assert_raises(ValueError, np.datetime64, 'now', 'D') + assert_raises(TypeError, np.datetime64, 'now', 'Y') + assert_raises(TypeError, np.datetime64, 'now', 'M') + assert_raises(TypeError, np.datetime64, 'now', 'D') def test_datetime_nat_casting(self): a = np.array('NaT', dtype='M8[D]') @@ -370,21 +425,14 @@ def test_dtype_comparison(self): assert_(np.dtype('M8[us]') != np.dtype('M8[ms]')) assert_(np.dtype('M8[2D]') != np.dtype('M8[D]')) assert_(np.dtype('M8[D]') != np.dtype('M8[2D]')) - assert_(np.dtype('M8[Y]//3') != np.dtype('M8[Y]')) def test_pydatetime_creation(self): a = np.array(['1960-03-12', datetime.date(1960, 3, 12)], dtype='M8[D]') assert_equal(a[0], a[1]) - a = np.array(['1960-03-12', datetime.date(1960, 3, 12)], dtype='M8[s]') - assert_equal(a[0], a[1]) a = np.array(['1999-12-31', datetime.date(1999, 12, 31)], dtype='M8[D]') assert_equal(a[0], a[1]) - a = np.array(['1999-12-31', datetime.date(1999, 12, 31)], dtype='M8[s]') - assert_equal(a[0], a[1]) a = np.array(['2000-01-01', datetime.date(2000, 1, 1)], dtype='M8[D]') assert_equal(a[0], a[1]) - a = np.array(['2000-01-01', datetime.date(2000, 1, 1)], dtype='M8[s]') - assert_equal(a[0], a[1]) # Will fail if the date changes during the exact right moment a = np.array(['today', datetime.date.today()], dtype='M8[D]') assert_equal(a[0], a[1]) @@ -392,9 +440,44 @@ def test_pydatetime_creation(self): #a = np.array(['now', datetime.datetime.now()], dtype='M8[s]') #assert_equal(a[0], a[1]) + # A datetime.date will raise if you try to give it time units + assert_raises(TypeError, np.array, datetime.date(1960, 3, 12), + dtype='M8[s]') + + def test_datetime_string_conversion(self): + a = ['2011-03-16', '1920-01-01', '2013-05-19'] + str_a = np.array(a, dtype='S') + dt_a = np.array(a, dtype='M') + str_b = np.empty_like(str_a) + dt_b = np.empty_like(dt_a) + + # String to datetime + assert_equal(dt_a, str_a.astype('M')) + assert_equal(dt_a.dtype, str_a.astype('M').dtype) + dt_b[...] = str_a + assert_equal(dt_a, dt_b) + # Datetime to string + assert_equal(str_a, dt_a.astype('S0')) + str_b[...] = dt_a + assert_equal(str_a, str_b) + + # Convert the 'S' to 'U' + str_a = str_a.astype('U') + str_b = str_b.astype('U') + + # Unicode to datetime + assert_equal(dt_a, str_a.astype('M')) + assert_equal(dt_a.dtype, str_a.astype('M').dtype) + dt_b[...] = str_a + assert_equal(dt_a, dt_b) + # Datetime to unicode + assert_equal(str_a, dt_a.astype('U')) + str_b[...] = dt_a + assert_equal(str_a, str_b) + def test_pickle(self): # Check that pickle roundtripping works - dt = np.dtype('M8[7D]//3') + dt = np.dtype('M8[7D]') assert_equal(dt, pickle.loads(pickle.dumps(dt))) dt = np.dtype('M8[W]') assert_equal(dt, pickle.loads(pickle.dumps(dt))) @@ -438,20 +521,35 @@ def test_pyobject_roundtrip(self): a = np.array([0,0,0,0,0,0,0,0,0, -1020040340, -2942398, -1, 0, 1, 234523453, 1199164176], dtype=np.int64) - for unit in ['M8[as]', 'M8[16fs]', 'M8[ps]', 'M8[us]', - 'M8[as]//12', 'M8[us]//16', 'M8[D]', 'M8[D]//4', - 'M8[W]', 'M8[M]', 'M8[Y]']: + # With date units + for unit in ['M8[D]', 'M8[W]', 'M8[M]', 'M8[Y]']: b = a.copy().view(dtype=unit) b[0] = '-0001-01-01' b[1] = '-0001-12-31' b[2] = '0000-01-01' b[3] = '0001-01-01' - b[4] = '1969-12-31T23:59:59.999999Z' + b[4] = '1969-12-31' b[5] = '1970-01-01' - b[6] = '9999-12-31T23:59:59.999999Z' + b[6] = '9999-12-31' b[7] = '10000-01-01' b[8] = 'NaT' + assert_equal(b.astype(object).astype(unit), b, + "Error roundtripping unit %s" % unit) + # With time units + for unit in ['M8[as]', 'M8[16fs]', 'M8[ps]', 'M8[us]', + 'M8[300as]', 'M8[20us]']: + b = a.copy().view(dtype=unit) + b[0] = '-0001-01-01T00Z' + b[1] = '-0001-12-31T00Z' + b[2] = '0000-01-01T00Z' + b[3] = '0001-01-01T00Z' + b[4] = '1969-12-31T23:59:59.999999Z' + b[5] = '1970-01-01T00Z' + b[6] = '9999-12-31T23:59:59.999999Z' + b[7] = '10000-01-01T00Z' + b[8] = 'NaT' + assert_equal(b.astype(object).astype(unit), b, "Error roundtripping unit %s" % unit) @@ -460,20 +558,19 @@ def test_month_truncation(self): assert_equal(np.array('1945-03-01', dtype='M8[M]'), np.array('1945-03-31', dtype='M8[M]')) assert_equal(np.array('1969-11-01', dtype='M8[M]'), - np.array('1969-11-30T23:59:59.999999Z', dtype='M8[M]')) + np.array('1969-11-30T23:59:59.99999Z', dtype='M').astype('M8[M]')) assert_equal(np.array('1969-12-01', dtype='M8[M]'), - np.array('1969-12-31T23:59:59.999999Z', dtype='M8[M]')) + np.array('1969-12-31T23:59:59.99999Z', dtype='M').astype('M8[M]')) assert_equal(np.array('1970-01-01', dtype='M8[M]'), - np.array('1970-01-31T23:59:59.999999Z', dtype='M8[M]')) + np.array('1970-01-31T23:59:59.99999Z', dtype='M').astype('M8[M]')) assert_equal(np.array('1980-02-01', dtype='M8[M]'), - np.array('1980-02-29T23:59:59.999999Z', dtype='M8[M]')) + np.array('1980-02-29T23:59:59.99999Z', dtype='M').astype('M8[M]')) def test_different_unit_comparison(self): - # Check some years with units that won't overflow - for unit1 in ['Y', 'M', 'D', '6h', 'h', 'm', 's', '10ms', - 'ms', 'us']: + # Check some years with date units + for unit1 in ['Y', 'M', 'D']: dt1 = np.dtype('M8[%s]' % unit1) - for unit2 in ['Y', 'M', 'D', 'h', 'm', 's', 'ms', 'us']: + for unit2 in ['Y', 'M', 'D']: dt2 = np.dtype('M8[%s]' % unit2) assert_equal(np.array('1945', dtype=dt1), np.array('1945', dtype=dt2)) @@ -491,19 +588,36 @@ def test_different_unit_comparison(self): np.datetime64('9999', unit2)) assert_equal(np.datetime64('10000', unit1), np.datetime64('10000-01-01', unit2)) + # Check some datetimes with time units + for unit1 in ['6h', 'h', 'm', 's', '10ms', 'ms', 'us']: + dt1 = np.dtype('M8[%s]' % unit1) + for unit2 in ['h', 'm', 's', 'ms', 'us']: + dt2 = np.dtype('M8[%s]' % unit2) + assert_equal(np.array('1945-03-12T18Z', dtype=dt1), + np.array('1945-03-12T18Z', dtype=dt2)) + assert_equal(np.array('1970-03-12T18Z', dtype=dt1), + np.array('1970-03-12T18Z', dtype=dt2)) + assert_equal(np.array('9999-03-12T18Z', dtype=dt1), + np.array('9999-03-12T18Z', dtype=dt2)) + assert_equal(np.array('10000-01-01T00Z', dtype=dt1), + np.array('10000-01-01T00Z', dtype=dt2)) + assert_equal(np.datetime64('1945-03-12T18Z', unit1), + np.datetime64('1945-03-12T18Z', unit2)) + assert_equal(np.datetime64('1970-03-12T18Z', unit1), + np.datetime64('1970-03-12T18Z', unit2)) + assert_equal(np.datetime64('9999-03-12T18Z', unit1), + np.datetime64('9999-03-12T18Z', unit2)) + assert_equal(np.datetime64('10000-01-01T00Z', unit1), + np.datetime64('10000-01-01T00Z', unit2)) # Check some days with units that won't overflow for unit1 in ['D', '12h', 'h', 'm', 's', '4s', 'ms', 'us']: dt1 = np.dtype('M8[%s]' % unit1) for unit2 in ['D', 'h', 'm', 's', 'ms', 'us']: dt2 = np.dtype('M8[%s]' % unit2) - assert_equal(np.array('1932-02-17', dtype=dt1), - np.array('1932-02-17T00:00:00Z', dtype=dt2)) - assert_equal(np.array('10000-04-27', dtype=dt1), - np.array('10000-04-27T00:00:00Z', dtype=dt2)) - assert_equal(np.datetime64('1932-02-17', unit1), - np.datetime64('1932-02-17T00:00:00Z', unit2)) - assert_equal(np.datetime64('10000-04-27', unit1), - np.datetime64('10000-04-27T00:00:00Z', unit2)) + assert_equal(np.array('1932-02-17', dtype='M').astype(dt1), + np.array('1932-02-17T00:00:00Z', dtype='M').astype(dt2)) + assert_equal(np.array('10000-04-27', dtype='M').astype(dt1), + np.array('10000-04-27T00:00:00Z', dtype='M').astype(dt2)) # Shouldn't be able to compare datetime and timedelta # TODO: Changing to 'same_kind' or 'safe' casting in the ufuncs by @@ -514,8 +628,8 @@ def test_different_unit_comparison(self): assert_raises(TypeError, np.less, a, b, casting='same_kind') def test_datetime_like(self): - a = np.array([3], dtype='m8[4D]//6') - b = np.array(['2012-12-21'], dtype='M8[D]//3') + a = np.array([3], dtype='m8[4D]') + b = np.array(['2012-12-21'], dtype='M8[D]') assert_equal(np.ones_like(a).dtype, a.dtype) assert_equal(np.zeros_like(a).dtype, a.dtype) @@ -633,7 +747,7 @@ def test_datetime_subtract(self): (np.array(['2012-12-21'], dtype='M8[D]'), np.array(['2012-12-24'], dtype='M8[D]'), np.array(['1940-12-24'], dtype='M8[D]'), - np.array(['1940-12-24'], dtype='M8[h]'), + np.array(['1940-12-24T00Z'], dtype='M8[h]'), np.array(['1940-12-23T13Z'], dtype='M8[h]'), np.array(['NaT'], dtype='M8[D]'), np.array([3], dtype='m8[D]'), @@ -643,7 +757,7 @@ def test_datetime_subtract(self): (np.datetime64('2012-12-21', '[D]'), np.datetime64('2012-12-24', '[D]'), np.datetime64('1940-12-24', '[D]'), - np.datetime64('1940-12-24', '[h]'), + np.datetime64('1940-12-24T00Z', '[h]'), np.datetime64('1940-12-23T13Z', '[h]'), np.datetime64('NaT', '[D]'), np.timedelta64(3, '[D]'), @@ -995,7 +1109,8 @@ def test_creation_overflow(self): def test_datetime_as_string(self): # Check all the units with default string conversion - date = '1959-10-13T12:34:56.789012345678901234Z' + date = '1959-10-13' + datetime = '1959-10-13T12:34:56.789012345678901234Z' assert_equal(np.datetime_as_string(np.datetime64(date, 'Y')), '1959') @@ -1003,50 +1118,54 @@ def test_datetime_as_string(self): '1959-10') assert_equal(np.datetime_as_string(np.datetime64(date, 'D')), '1959-10-13') - assert_equal(np.datetime_as_string(np.datetime64(date, 'h')), + assert_equal(np.datetime_as_string(np.datetime64(datetime, 'h')), '1959-10-13T12Z') - assert_equal(np.datetime_as_string(np.datetime64(date, 'm')), + assert_equal(np.datetime_as_string(np.datetime64(datetime, 'm')), '1959-10-13T12:34Z') - assert_equal(np.datetime_as_string(np.datetime64(date, 's')), + assert_equal(np.datetime_as_string(np.datetime64(datetime, 's')), '1959-10-13T12:34:56Z') - assert_equal(np.datetime_as_string(np.datetime64(date, 'ms')), + assert_equal(np.datetime_as_string(np.datetime64(datetime, 'ms')), '1959-10-13T12:34:56.789Z') - assert_equal(np.datetime_as_string(np.datetime64(date, 'us')), + assert_equal(np.datetime_as_string(np.datetime64(datetime, 'us')), '1959-10-13T12:34:56.789012Z') - date = '1969-12-31T23:34:56.789012345678901234Z' + datetime = '1969-12-31T23:34:56.789012345678901234Z' - assert_equal(np.datetime_as_string(np.datetime64(date, 'ns')), + assert_equal(np.datetime_as_string(np.datetime64(datetime, 'ns')), '1969-12-31T23:34:56.789012345Z') - assert_equal(np.datetime_as_string(np.datetime64(date, 'ps')), + assert_equal(np.datetime_as_string(np.datetime64(datetime, 'ps')), '1969-12-31T23:34:56.789012345678Z') - assert_equal(np.datetime_as_string(np.datetime64(date, 'fs')), + assert_equal(np.datetime_as_string(np.datetime64(datetime, 'fs')), '1969-12-31T23:34:56.789012345678901Z') - date = '1969-12-31T23:59:57.789012345678901234Z' + datetime = '1969-12-31T23:59:57.789012345678901234Z' - assert_equal(np.datetime_as_string(np.datetime64(date, 'as')), - date) - date = '1970-01-01T00:34:56.789012345678901234Z' + assert_equal(np.datetime_as_string(np.datetime64(datetime, 'as')), + datetime) + datetime = '1970-01-01T00:34:56.789012345678901234Z' - assert_equal(np.datetime_as_string(np.datetime64(date, 'ns')), + assert_equal(np.datetime_as_string(np.datetime64(datetime, 'ns')), '1970-01-01T00:34:56.789012345Z') - assert_equal(np.datetime_as_string(np.datetime64(date, 'ps')), + assert_equal(np.datetime_as_string(np.datetime64(datetime, 'ps')), '1970-01-01T00:34:56.789012345678Z') - assert_equal(np.datetime_as_string(np.datetime64(date, 'fs')), + assert_equal(np.datetime_as_string(np.datetime64(datetime, 'fs')), '1970-01-01T00:34:56.789012345678901Z') - date = '1970-01-01T00:00:05.789012345678901234Z' + datetime = '1970-01-01T00:00:05.789012345678901234Z' - assert_equal(np.datetime_as_string(np.datetime64(date, 'as')), - date) + assert_equal(np.datetime_as_string(np.datetime64(datetime, 'as')), + datetime) # String conversion with the unit= parameter a = np.datetime64('2032-07-18T12:23:34.123456Z', 'us') - assert_equal(np.datetime_as_string(a, unit='Y'), '2032') - assert_equal(np.datetime_as_string(a, unit='M'), '2032-07') - assert_equal(np.datetime_as_string(a, unit='W'), '2032-07-18') - assert_equal(np.datetime_as_string(a, unit='D'), '2032-07-18') + assert_equal(np.datetime_as_string(a, unit='Y', casting='unsafe'), + '2032') + assert_equal(np.datetime_as_string(a, unit='M', casting='unsafe'), + '2032-07') + assert_equal(np.datetime_as_string(a, unit='W', casting='unsafe'), + '2032-07-18') + assert_equal(np.datetime_as_string(a, unit='D', casting='unsafe'), + '2032-07-18') assert_equal(np.datetime_as_string(a, unit='h'), '2032-07-18T12Z') assert_equal(np.datetime_as_string(a, unit='m'), '2032-07-18T12:23Z') @@ -1101,16 +1220,47 @@ def test_datetime_as_string(self): unit='auto'), '2032-01-01') - # local=True + def test_datetime_as_string_timezone(self): + # timezone='local' vs 'UTC' a = np.datetime64('2010-03-15T06:30Z', 'm') - assert_(np.datetime_as_string(a, local=True) != '2010-03-15T6:30Z') - # local=True with tzoffset - assert_equal(np.datetime_as_string(a, local=True, tzoffset=-60), - '2010-03-15T05:30-0100') - assert_equal(np.datetime_as_string(a, local=True, tzoffset=+30), - '2010-03-15T07:00+0030') - assert_equal(np.datetime_as_string(a, local=True, tzoffset=-5*60), - '2010-03-15T01:30-0500') + assert_equal(np.datetime_as_string(a, timezone='UTC'), + '2010-03-15T06:30Z') + assert_(np.datetime_as_string(a, timezone='local') != + '2010-03-15T06:30Z') + + # Use pytz to test out various time zones + try: + from pytz import timezone as tz + + b = np.datetime64('2010-02-15T06:30Z', 'm') + + assert_equal(np.datetime_as_string(a, timezone=tz('US/Central')), + '2010-03-15T01:30-0500') + assert_equal(np.datetime_as_string(a, timezone=tz('US/Eastern')), + '2010-03-15T02:30-0400') + assert_equal(np.datetime_as_string(a, timezone=tz('US/Pacific')), + '2010-03-14T23:30-0700') + + assert_equal(np.datetime_as_string(b, timezone=tz('US/Central')), + '2010-02-15T00:30-0600') + assert_equal(np.datetime_as_string(b, timezone=tz('US/Eastern')), + '2010-02-15T01:30-0500') + assert_equal(np.datetime_as_string(b, timezone=tz('US/Pacific')), + '2010-02-14T22:30-0800') + + # Dates to strings with a timezone attached is disabled by default + assert_raises(TypeError, np.datetime_as_string, a, unit='D', + timezone=tz('US/Pacific')) + # Check that we can print out the date in the specified time zone + assert_equal(np.datetime_as_string(a, unit='D', + timezone=tz('US/Pacific'), casting='unsafe'), + '2010-03-14') + assert_equal(np.datetime_as_string(b, unit='D', + timezone=tz('US/Central'), casting='unsafe'), + '2010-02-15') + except ImportError: + import warnings + warnings.warn("Need pytz library to test datetime timezones") def test_datetime_arange(self): # With two datetimes provided as strings @@ -1460,7 +1610,7 @@ class TestDateTimeData(TestCase): def test_basic(self): a = np.array(['1980-03-23'], dtype=np.datetime64) - assert_equal(np.datetime_data(a.dtype), (asbytes('D'), 1, 1)) + assert_equal(np.datetime_data(a.dtype), (asbytes('D'), 1)) if __name__ == "__main__": run_module_suite()