8000 feat: add decimal/numeric support (#620) · googleapis/python-spanner-django@d09ad61 · GitHub
[go: up one dir, main page]
Skip to content

Commit d09ad61

Browse files
vi3k6i5vi3k6i5
authored
feat: add decimal/numeric support (#620)
* fix: lint_setup_py was failing in Kokoro is not fixed * feat: add decimal/numeric support * fix: remove validation for decimal field not supported * feat: updated decimal support error message in spanner to match error thrown by python spanner decimal/numeric validation * fix: removed test_validation as decimal support is now added so validation is not required * fix: Remove system tests. They will be added separately. * fix: fixed tests related to decimal conversion in db operations * fix: fixed tests related to decimal conversion in db operations * refactor: lint corrections in test_operations file * fix: corrected coverage number, lowered it t 65 * refactor: lint issues fixed in noxfile and import moved up to module level in test_lookups
1 parent 92ad508 commit d09ad61

File tree

10 files changed

+34
-140
lines changed

10 files changed

+34
-140
lines changed

‎django_spanner/base.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
from .introspection import DatabaseIntrospection
1818
from .operations import DatabaseOperations
1919
from .schema import DatabaseSchemaEditor
20-
from .validation import DatabaseValidation
2120

2221

2322
class DatabaseWrapper(BaseDatabaseWrapper):
@@ -34,7 +33,7 @@ class DatabaseWrapper(BaseDatabaseWrapper):
3433
"CharField": "STRING(%(max_length)s)",
3534
"DateField": "DATE",
3635
"DateTimeField": "TIMESTAMP",
37-
"DecimalField": "FLOAT64",
36+
"DecimalField": "NUMERIC",
3837
"DurationField": "INT64",
3938
"EmailField": "STRING(%(max_length)s)",
4039
"FileField": "STRING(%(max_length)s)",
@@ -104,7 +103,6 @@ class DatabaseWrapper(BaseDatabaseWrapper):
104103
introspection_class = DatabaseIntrospection
105104
ops_class = DatabaseOperations
106105
client_class = DatabaseClient
107-
validation_class = DatabaseValidation
108106

109107
@property
110108
def instance(self):

‎django_spanner/features.py

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -233,10 +233,6 @@ class DatabaseFeatures(BaseDatabaseFeatures):
233233
"queries.test_bulk_update.BulkUpdateTests.test_large_batch",
234234
# Spanner doesn't support random ordering.
235235
"ordering.tests.OrderingTests.test_random_ordering",
236-
# No matching signature for function MOD for argument types: FLOAT64,
237-
# FLOAT64. Supported signatures: MOD(INT64, INT64)
238-
"db_functions.math.test_mod.ModTests.test_decimal",
239-
"db_functions.math.test_mod.ModTests.test_float",
240236
# casting DateField to DateTimeField adds an unexpected hour:
241237
# https://github.com/googleapis/python-spanner-django/issues/260
242238
"db_functions.comparison.test_cast.CastTests.test_cast_from_db_date_to_date EDBE time",
@@ -364,6 +360,11 @@ class DatabaseFeatures(BaseDatabaseFeatures):
364360
"model_formsets.tests.ModelFormsetTest.test_prevent_change_outer_model_and_create_invalid_data",
365361
"model_formsets_regress.tests.FormfieldShouldDeleteFormTests.test_no_delete",
366362
"model_formsets_regress.tests.FormsetTests.test_extraneous_query_is_not_run",
363+
# Numeric field is not supported in primary key/unique key.
364+
"model_formsets.tests.ModelFormsetTest.test_inline_formsets_with_custom_pk",
365+
"model_forms.tests.ModelFormBaseTest.test_exclude_and_validation",
366+
"model_forms.tests.UniqueTest.test_unique_together",
367+
"model_forms.tests.UniqueTest.test_override_unique_together_message",
367368
# os.chmod() doesn't work on Kokoro?
368369
"file_uploads.tests.DirectoryCreationTests.test_readonly_root",
369370
# Tests that sometimes fail on Kokoro for unknown reasons.
@@ -1026,12 +1027,20 @@ class DatabaseFeatures(BaseDatabaseFeatures):
10261027
"db_functions.math.test_ceil.CeilTests.test_null", # noqa
10271028
"db_functions.math.test_ceil.CeilTests.test_transform", # noqa
10281029
"db_functions.math.test_cos.CosTests.test_null", # noqa
1030+
"db_functions.math.test_cos.CosTests.test_transform", # noqa
10291031
"db_functions.math.test_cot.CotTests.test_null", # noqa
1032+
"db_functions.math.test_degrees.DegreesTests.test_decimal", # noqa
10301033
"db_functions.math.test_degrees.DegreesTests.test_null", # noqa
1034+
"db_functions.math.test_exp.ExpTests.test_decimal", # noqa
10311035
"db_functions.math.test_exp.ExpTests.test_null", # noqa
1036+
"db_functions.math.test_exp.ExpTests.test_transform", # noqa
10321037
"db_functions.math.test_floor.FloorTests.test_null", # noqa
1038+
"db_functions.math.test_ln.LnTests.test_decimal", # noqa
10331039
"db_functions.math.test_ln.LnTests.test_null", # noqa
1040+
"db_functions.math.test_ln.LnTests.test_transform", # noqa
1041+
"db_functions.math.test_log.LogTests.test_decimal", # noqa
10341042
"db_functions.math.test_log.LogTests.test_null", # noqa
1043+
"db_functions.math.test_mod.ModTests.test_float", # noqa
10351044
"db_functions.math.test_mod.ModTests.test_null", # noqa
10361045
"db_functions.math.test_power.PowerTests.test_decimal", # noqa
10371046
"db_functions.math.test_power.PowerTests.test_float", # noqa
@@ -1040,7 +1049,9 @@ class DatabaseFeatures(BaseDatabaseFeatures):
10401049
"db_functions.math.test_radians.RadiansTests.test_null", # noqa
10411050
"db_functions.math.test_round.RoundTests.test_null", # noqa
10421051
"db_functions.math.test_sin.SinTests.test_null", # noqa
1052+
"db_functions.math.test_sqrt.SqrtTests.test_decimal", # noqa
10431053
"db_functions.math.test_sqrt.SqrtTests.test_null", # noqa
1054+
"db_functions.math.test_sqrt.SqrtTests.test_transform", # noqa
10441055
"db_functions.math.test_tan.TanTests.test_null", # noqa
10451056
"db_functions.tests.FunctionTests.test_func_transform_bilateral", # noqa
10461057
"db_functions.tests.FunctionTests.test_func_transform_bilateral_multivalue", # noqa

‎django_spanner/introspection.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ class DatabaseIntrospection(BaseDatabaseIntrospection):
2424
TypeCode.INT64: "IntegerField",
2525
TypeCode.STRING: "CharField",
2626
TypeCode.TIMESTAMP: "DateTimeField",
27+
TypeCode.NUMERIC: "DecimalField",
2728
}
2829

2930
def get_field_type(self, data_type, description):

‎django_spanner/lookups.py

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
# license that can be found in the LICENSE file or at
55
# https://developers.google.com/open-source/licenses/bsd
66

7-
from django.db.models import DecimalField
87
from django.db.models.lookups import (
98
Contains,
109
EndsWith,
@@ -233,13 +232,8 @@ def cast_param_to_float(self, compiler, connection):
233232
"""
234233
sql, params = self.as_sql(compiler, connection)
235234
if params:
236-
# Cast to DecimaField lookup values to float because
237-
# google.cloud.spanner_v1._helpers._make_value_pb() doesn't serialize
238-
# decimal.Decimal.
239-
if isinstance(self.lhs.output_field, DecimalField):
240-
params[0] = float(params[0])
241235
# Cast remote field lookups that must be integer but come in as string.
242-
elif hasattr(self.lhs.output_field, "get_path_info"):
236+
if hasattr(self.lhs.output_field, "get_path_info"):
243237
for i, field in enumerate(
244238
self.lhs.output_field.get_path_info()[-1].target_fields
245239
):

‎django_spanner/operations.py

Lines changed: 7 additions & 31 deletions
10000
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
import re
99
from base64 import b64decode
1010
from datetime import datetime, time
11-
from decimal import Decimal
1211
from uuid import UUID
1312

1413
from django.conf import settings
@@ -190,10 +189,11 @@ def adapt_decimalfield_value(
190189
self, value, max_digits=None, decimal_places=None
191190
):
192191
"""
193-
Convert value from decimal.Decimal into float, for a direct mapping
194-
and correct serialization with RPCs to Cloud Spanner.
192+
Convert value from decimal.Decimal to spanner compatible value.
193+
Since spanner supports Numeric storage of decimal and python spanner
194+
takes care of the conversion so this is a no-op method call.
195195
196-
:type value: :class:`~google.cloud.spanner_v1.types.Numeric`
196+
:type value: :class:`decimal.Decimal`
197197
:param value: A decimal field value.
198198
199199
:type max_digits: int
@@ -203,12 +203,10 @@ def adapt_decimalfield_value(
203203
:param decimal_places: (Optional) The number of decimal places to store
204204
with the number.
205205
206-
:rtype: float
207-
:returns: Formatted value.
206+
:rtype: decimal.Decimal
207+
:returns: decimal value.
208208
"""
209-
if value is None:
210-
return None
211-
return float(value)
209+
return value
212210

213211
def adapt_timefield_value(self, value):
214212
"""
@@ -244,8 +242,6 @@ def get_db_converters(self, expression):
244242
internal_type = expression.output_field.get_internal_type()
245243
if internal_type == "DateTimeField":
246244
converters.append(self.convert_datetimefield_value)
247-
elif internal_type == "DecimalField":
248-
converters.append(self.convert_decimalfield_value)
249245
elif internal_type == "TimeField":
250246
converters.append(self.convert_timefield_value)
251247
elif internal_type == "BinaryField":
@@ -311,26 +307,6 @@ def convert_datetimefield_value(self, value, expression, connection):
311307
else dt
312308
)
313309

314-
def convert_decimalfield_value(self, value, expression, connection):
315-
"""Convert Spanner DecimalField value for Django.
316-
317-
:type value: float
318-
:param value: A decimal field.
319-
320-
:type expression: :class:`django.db.models.expressions.BaseExpression`
321-
:param expression: A query expression.
322-
323-
:type connection: :class:`~google.cloud.cpanner_dbapi.connection.Connection`
324-
:param connection: Reference to a Spanner database connection.
325-
326-
:rtype: :class:`Decimal`
327-
:returns: A converted decimal field.
328-
"""
329-
if value is None:
330-
return value
331-
# Cloud Spanner returns a float.
332-
return Decimal(str(value))
333-
334310
def convert_timefield_value(self, value, expression, connection):
335311
"""Convert Spanner TimeField value for Django.
336312

‎django_spanner/validation.py

Lines changed: 0 additions & 33 deletions
This file was deleted.

‎noxfile.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ def default(session):
8484
"--cov-append",
8585
"--cov-config=.coveragerc",
8686
"--cov-report=",
87-
"--cov-fail-under=68",
87+
"--cov-fail-under=65",
8888
os.path.join("tests", "unit"),
8989
*session.posargs
9090
)

‎tests/unit/django_spanner/test_lookups.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,21 +7,24 @@
77
from django_spanner.compiler import SQLCompiler
88
from django.db.models import F
99
from tests.unit.django_spanner.simple_test import SpannerSimpleTestClass
10+
from decimal import Decimal
1011
from .models import Number, Author
1112

1213

1314
class TestLookups(SpannerSimpleTestClass):
1415
def test_cast_param_to_float_lte_sql_query(self):
1516

16-
qs1 = Number.objects.filter(decimal_num__lte=1.1).values("decimal_num")
17+
qs1 = Number.objects.filter(decimal_num__lte=Decimal("1.1")).values(
18+
"decimal_num"
19+
)
1720
compiler = SQLCompiler(qs1.query, self.connection, "default")
1821
sql_compiled, params = compiler.as_sql()
1922
self.assertEqual(
2023
sql_compiled,
2124
"SELECT tests_number.decimal_num FROM tests_number WHERE "
2225
+ "tests_number.decimal_num <= %s",
2326
)
24-
self.assertEqual(params, (1.1,))
27+
self.assertEqual(params, (Decimal("1.1"),))
2528

2629
def test_cast_param_to_float_for_int_field_query(self):
2730

‎tests/unit/django_spanner/test_operations.py

Lines changed: 3 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from django.db.utils import DatabaseError
88
from datetime import timedelta
99
from tests.unit.django_spanner.simple_test import SpannerSimpleTestClass
10+
from decimal import Decimal
1011

1112

1213
class TestOperations(SpannerSimpleTestClass):
@@ -58,7 +59,8 @@ def test_adapt_datefield_value_none(self):
5859

5960
def test_adapt_decimalfield_value(self):
6061
self.assertIsInstance(
61-
self.db_operations.adapt_decimalfield_value(value=1), float,
62+
self.db_operations.adapt_decimalfield_value(value=Decimal("1")),
63+
Decimal,
6264
)
6365

6466
def test_adapt_decimalfield_value_none(self):
@@ -93,23 +95,6 @@ def test_adapt_timefield_value_none(self):
9395
self.db_operations.adapt_timefield_value(value=None),
9496
)
9597

96-
def test_convert_decimalfield_value(self):
97-
from decimal import Decimal
98-
99-
self.assertIsInstance(
100-
self.db_operations.convert_decimalfield_value(
101-
value=1.0, expression=None, connection=None
102-
),
103-
Decimal,
104-
)
105-
106-
def test_convert_decimalfield_value_none(self):
107-
self.assertIsNone(
108-
self.db_operations.convert_decimalfield_value(
109-
value=None, expression=None, connection=None
110-
),
111-
)
112-
11398
def test_convert_uuidfield_value(self):
11499
import uuid
115100

‎tests/unit/django_spanner/test_validation.py

Lines changed: 0 additions & 41 deletions
This file was deleted.

0 commit comments

Comments
 (0)
0