From 105d4a02889a85d21d8d4d59dba8262acf06b233 Mon Sep 17 00:00:00 2001 From: Mariatta Wijaya Date: Mon, 27 Mar 2023 20:13:47 -0700 Subject: [PATCH 01/18] feat: Add support for Sum and Avg aggregation query Add .sum() and .avg() functions to aggregation Refactor limit to be passed in to the nested query's limit Unit tests --- google/cloud/datastore/aggregation.py | 85 +++++++++++++++++++- tests/system/test_aggregation_query.py | 85 ++++++++++++++++++-- tests/unit/test_aggregation.py | 107 ++++++++++++++++++++++++- 3 files changed, 264 insertions(+), 13 deletions(-) diff --git a/google/cloud/datastore/aggregation.py b/google/cloud/datastore/aggregation.py index 24d2abcc..f62a76ad 100644 --- a/google/cloud/datastore/aggregation.py +++ b/google/cloud/datastore/aggregation.py @@ -39,6 +39,9 @@ class BaseAggregation(ABC): Base class representing an Aggregation operation in Datastore """ + def __init__(self, alias=None): + self.alias = alias + @abc.abstractmethod def _to_pb(self): """ @@ -59,7 +62,7 @@ class CountAggregation(BaseAggregation): """ def __init__(self, alias=None): - self.alias = alias + super(CountAggregation, self).__init__(alias=alias) def _to_pb(self): """ @@ -71,6 +74,61 @@ def _to_pb(self): return aggregation_pb +class SumAggregation(BaseAggregation): + """ + Representation of a "Sum" aggregation query. + + :type property_ref: str + :param property_ref: The property_ref for the aggregation. + + :type value: int + :param value: The resulting value from the aggregation. + + """ + + def __init__(self, property_ref, alias=None): + self.property_ref = property_ref + super(SumAggregation, self).__init__(alias=alias) + + def _to_pb(self): + """ + Convert this instance to the protobuf representation + """ + aggregation_pb = query_pb2.AggregationQuery.Aggregation() + aggregation_pb.alias = self.alias + aggregation_pb.sum_ = query_pb2.AggregationQuery.Aggregation.Sum() + aggregation_pb.sum_.property.name = self.property_ref + aggregation_pb.alias = self.alias + return aggregation_pb + + +class AvgAggregation(BaseAggregation): + """ + Representation of a "Avg" aggregation query. + + :type property_ref: str + :param property_ref: The property_ref for the aggregation. + + :type value: int + :param value: The resulting value from the aggregation. + + """ + + def __init__(self, property_ref, alias=None): + self.property_ref = property_ref + super(AvgAggregation, self).__init__(alias=alias) + + def _to_pb(self): + """ + Convert this instance to the protobuf representation + """ + aggregation_pb = query_pb2.AggregationQuery.Aggregation() + aggregation_pb.avg = query_pb2.AggregationQuery.Aggregation.Avg() + aggregation_pb.avg.property.name = self.property_ref + aggregation_pb.alias = self.alias + return aggregation_pb + + class AggregationResult(object): """ A class representing result from Aggregation Query @@ -154,6 +212,28 @@ def count(self, alias=None): self._aggregations.append(count_aggregation) return self + def sum(self, property_ref, alias=None): + """ + Adds a sum over the nested query + + :type property_ref: str + :param property_ref: The property_ref for the sum + """ + sum_aggregation = SumAggregation(property_ref=property_ref, alias=alias) + self._aggregations.append(sum_aggregation) + return self + + def avg(self, property_ref, alias=None): + """ + Adds a avg over the nested query + + :type property_ref: str + :param property_ref: The property_ref for the sum + """ + avg_aggregation = AvgAggregation(property_ref=property_ref, alias=alias) + self._aggregations.append(avg_aggregation) + return self + def add_aggregation(self, aggregation): """ Adds an aggregation operation to the nested query @@ -327,8 +407,7 @@ def _build_protobuf(self): """ pb = self._aggregation_query._to_pb() if self._limit is not None and self._limit > 0: - for aggregation in pb.aggregations: - aggregation.count.up_to = self._limit + pb.nested_query.limit = self._limit return pb def _process_query_results(self, response_pb): diff --git a/tests/system/test_aggregation_query.py b/tests/system/test_aggregation_query.py index b912e96b..1fc54ddb 100644 --- a/tests/system/test_aggregation_query.py +++ b/tests/system/test_aggregation_query.py @@ -93,6 +93,54 @@ def test_aggregation_query_with_alias(aggregation_query_client, nested_query): assert r.value > 0 +def test_sum_query_default(aggregation_query_client, nested_query): + query = nested_query + + aggregation_query = aggregation_query_client.aggregation_query(query) + aggregation_query.sum("person") + result = _do_fetch(aggregation_query) + assert len(result) == 1 + for r in result[0]: + assert r.alias == "property_1" + assert r.value == 8 + + +def test_sum_query_with_alias(aggregation_query_client, nested_query): + query = nested_query + + aggregation_query = aggregation_query_client.aggregation_query(query) + aggregation_query.sum("person", alias="sum_person") + result = _do_fetch(aggregation_query) + assert len(result) == 1 + for r in result[0]: + assert r.alias == "sum_person" + assert r.value > 0 + + +def test_avg_query_default(aggregation_query_client, nested_query): + query = nested_query + + aggregation_query = aggregation_query_client.aggregation_query(query) + aggregation_query.avg("person") + result = _do_fetch(aggregation_query) + assert len(result) == 1 + for r in result[0]: + assert r.alias == "property_1" + assert r.value == 8 + + +def test_avg_query_with_alias(aggregation_query_client, nested_query): + query = nested_query + + aggregation_query = aggregation_query_client.aggregation_query(query) + aggregation_query.avg("person", alias="avg_person") + result = _do_fetch(aggregation_query) + assert len(result) == 1 + for r in result[0]: + assert r.alias == "avg_person" + assert r.value > 0 + + def test_aggregation_query_with_limit(aggregation_query_client, nested_query): query = nested_query @@ -121,41 +169,60 @@ def test_aggregation_query_multiple_aggregations( aggregation_query = aggregation_query_client.aggregation_query(query) aggregation_query.count(alias="total") aggregation_query.count(alias="all") + aggregation_query.sum("person", alias="sum_person") + aggregation_query.avg("person", alias="avg_person") result = _do_fetch(aggregation_query) assert len(result) == 1 for r in result[0]: - assert r.alias in ["all", "total"] + assert r.alias in ["all", "total", "sum_person", "avg_person"] assert r.value > 0 def test_aggregation_query_add_aggregation(aggregation_query_client, nested_query): from google.cloud.datastore.aggregation import CountAggregation + from google.cloud.datastore.aggregation import SumAggregation + from google.cloud.datastore.aggregation import AvgAggregation query = nested_query aggregation_query = aggregation_query_client.aggregation_query(query) count_aggregation = CountAggregation(alias="total") aggregation_query.add_aggregation(count_aggregation) + + sum_aggregation = SumAggregation("person", alias="sum_person") + aggregation_query.add_aggregation(sum_aggregation) + + avg_aggregation = AvgAggregation("person", alias="avg_person") + aggregation_query.add_aggregation(avg_aggregation) + result = _do_fetch(aggregation_query) assert len(result) == 1 for r in result[0]: - assert r.alias == "total" + assert r.alias in ["total", "sum_person", "avg_person"] assert r.value > 0 def test_aggregation_query_add_aggregations(aggregation_query_client, nested_query): - from google.cloud.datastore.aggregation import CountAggregation + from google.cloud.datastore.aggregation import ( + CountAggregation, + SumAggregation, + AvgAggregation, + ) query = nested_query aggregation_query = aggregation_query_client.aggregation_query(query) count_aggregation_1 = CountAggregation(alias="total") count_aggregation_2 = CountAggregation(alias="all") - aggregation_query.add_aggregations([count_aggregation_1, count_aggregation_2]) + sum_aggregation = SumAggregation("person", alias="sum_person") + avg_aggregation = AvgAggregation("person", alias="avg_person") + aggregation_query.add_aggregations( + [count_aggregation_1, count_aggregation_2, sum_aggregation, avg_aggregation] + ) result = _do_fetch(aggregation_query) assert len(result) == 1 for r in result[0]: - assert r.alias in ["total", "all"] + assert r.alias in ["total", "all", "sum_person", "avg_person"] assert r.value > 0 @@ -202,11 +269,13 @@ def test_aggregation_query_with_nested_query_filtered( aggregation_query = aggregation_query_client.aggregation_query(query) aggregation_query.count(alias="total") + aggregation_query.sum("person", alias="sum_person") + aggregation_query.avg("person", alias="avg_person") result = _do_fetch(aggregation_query) assert len(result) == 1 for r in result[0]: - assert r.alias == "total" + assert r.alias in ["total", "sum_person", "avg_person"] assert r.value == 6 @@ -226,9 +295,11 @@ def test_aggregation_query_with_nested_query_multiple_filters( aggregation_query = aggregation_query_client.aggregation_query(query) aggregation_query.count(alias="total") + aggregation_query.sum("person", alias="sum_person") + aggregation_query.avg("person", alias="avg_person") result = _do_fetch(aggregation_query) assert len(result) == 1 for r in result[0]: - assert r.alias == "total" + assert r.alias in ["total", "sum_person", "avg_person"] assert r.value == 4 diff --git a/tests/unit/test_aggregation.py b/tests/unit/test_aggregation.py index afa9dc53..afb06697 100644 --- a/tests/unit/test_aggregation.py +++ b/tests/unit/test_aggregation.py @@ -15,7 +15,12 @@ import mock import pytest -from google.cloud.datastore.aggregation import CountAggregation, AggregationQuery +from google.cloud.datastore.aggregation import ( + CountAggregation, + SumAggregation, + AvgAggregation, + AggregationQuery, +) from tests.unit.test_query import _make_query, _make_client @@ -33,6 +38,30 @@ def test_count_aggregation_to_pb(): assert count_aggregation._to_pb() == expected_aggregation_query_pb +def test_sum_aggregation_to_pb(): + from google.cloud.datastore_v1.types import query as query_pb2 + + sum_aggregation = SumAggregation("person", alias="total") + + expected_aggregation_query_pb = query_pb2.AggregationQuery.Aggregation() + expected_aggregation_query_pb.sum_ = query_pb2.AggregationQuery.Aggregation.Sum() + expected_aggregation_query_pb.sum_.property.name = sum_aggregation.property_ref + expected_aggregation_query_pb.alias = sum_aggregation.alias + assert sum_aggregation._to_pb() == expected_aggregation_query_pb + + +def test_avg_aggregation_to_pb(): + from google.cloud.datastore_v1.types import query as query_pb2 + + avg_aggregation = AvgAggregation("person", alias="total") + + expected_aggregation_query_pb = query_pb2.AggregationQuery.Aggregation() + expected_aggregation_query_pb.avg = query_pb2.AggregationQuery.Aggregation.Avg() + expected_aggregation_query_pb.avg.property.name = avg_aggregation.property_ref + expected_aggregation_query_pb.alias = avg_aggregation.alias + assert avg_aggregation._to_pb() == expected_aggregation_query_pb + + @pytest.fixture def client(): return _make_client() @@ -80,6 +109,8 @@ def test_pb_over_query_with_add_aggregations(client): aggregations = [ CountAggregation(alias="total"), CountAggregation(alias="all"), + SumAggregation("person", alias="sum_person"), + AvgAggregation("person", alias="avg_person"), ] query = _make_query(client) @@ -88,9 +119,63 @@ def test_pb_over_query_with_add_aggregations(client): aggregation_query.add_aggregations(aggregations) pb = aggregation_query._to_pb() assert pb.nested_query == _pb_from_query(query) - assert len(pb.aggregations) == 2 + assert len(pb.aggregations) == 4 assert pb.aggregations[0] == CountAggregation(alias="total")._to_pb() assert pb.aggregations[1] == CountAggregation(alias="all")._to_pb() + assert pb.aggregations[2] == SumAggregation("person", alias="sum_person")._to_pb() + assert pb.aggregations[3] == AvgAggregation("person", alias="avg_person")._to_pb() + + +def test_pb_over_query_with_sum(client): + from google.cloud.datastore.query import _pb_from_query + + query = _make_query(client) + aggregation_query = _make_aggregation_query(client=client, query=query) + + aggregation_query.sum("person", alias="total") + pb = aggregation_query._to_pb() + assert pb.nested_query == _pb_from_query(query) + assert len(pb.aggregations) == 1 + assert pb.aggregations[0] == SumAggregation("person", alias="total")._to_pb() + + +def test_pb_over_query_sum_with_add_aggregation(client): + from google.cloud.datastore.query import _pb_from_query + + query = _make_query(client) + aggregation_query = _make_aggregation_query(client=client, query=query) + + aggregation_query.add_aggregation(SumAggregation("person", alias="total")) + pb = aggregation_query._to_pb() + assert pb.nested_query == _pb_from_query(query) + assert len(pb.aggregations) == 1 + assert pb.aggregations[0] == SumAggregation("person", alias="total")._to_pb() + + +def test_pb_over_query_with_avg(client): + from google.cloud.datastore.query import _pb_from_query + + query = _make_query(client) + aggregation_query = _make_aggregation_query(client=client, query=query) + + aggregation_query.avg("person", alias="total") + pb = aggregation_query._to_pb() + assert pb.nested_query == _pb_from_query(query) + assert len(pb.aggregations) == 1 + assert pb.aggregations[0] == AvgAggregation("person", alias="total")._to_pb() + + +def test_pb_over_query_avf_with_add_aggregation(client): + from google.cloud.datastore.query import _pb_from_query + + query = _make_query(client) + aggregation_query = _make_aggregation_query(client=client, query=query) + + aggregation_query.add_aggregation(AvgAggregation("person", alias="total")) + pb = aggregation_query._to_pb() + assert pb.nested_query == _pb_from_query(query) + assert len(pb.aggregations) == 1 + assert pb.aggregations[0] == AvgAggregation("person", alias="total")._to_pb() def test_query_fetch_defaults_w_client_attr(client): @@ -203,8 +288,11 @@ def test_iterator__build_protobuf_all_values(): query = _make_query(client) alias = "total" limit = 2 + property_ref = "person" aggregation_query = AggregationQuery(client=client, query=query) aggregation_query.count(alias) + aggregation_query.sum(property_ref) + aggregation_query.avg(property_ref) iterator = _make_aggregation_iterator(aggregation_query, client, limit=limit) iterator.num_results = 4 @@ -212,9 +300,22 @@ def test_iterator__build_protobuf_all_values(): pb = iterator._build_protobuf() expected_pb = query_pb2.AggregationQuery() expected_pb.nested_query = query_pb2.Query() + expected_pb.nested_query.limit = limit + expected_count_pb = query_pb2.AggregationQuery.Aggregation(alias=alias) - expected_count_pb.count.up_to = limit + expected_count_pb.count = query_pb2.AggregationQuery.Aggregation.Count() expected_pb.aggregations.append(expected_count_pb) + + expected_sum_pb = query_pb2.AggregationQuery.Aggregation() + expected_sum_pb.sum_ = query_pb2.AggregationQuery.Aggregation.Sum() + expected_sum_pb.sum_.property.name = property_ref + expected_pb.aggregations.append(expected_sum_pb) + + expected_avg_pb = query_pb2.AggregationQuery.Aggregation() + expected_avg_pb.avg = query_pb2.AggregationQuery.Aggregation.Avg() + expected_avg_pb.avg.property.name = property_ref + expected_pb.aggregations.append(expected_avg_pb) + assert pb == expected_pb From 77b17a5b2e98fe98326b9a07aa57edf646685d86 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 18 Aug 2023 15:21:45 -0700 Subject: [PATCH 02/18] fixed sum_ proto field --- google/cloud/datastore/aggregation.py | 4 ++-- tests/unit/test_aggregation.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/google/cloud/datastore/aggregation.py b/google/cloud/datastore/aggregation.py index f091beff..360d055a 100644 --- a/google/cloud/datastore/aggregation.py +++ b/google/cloud/datastore/aggregation.py @@ -96,8 +96,8 @@ def _to_pb(self): """ aggregation_pb = query_pb2.AggregationQuery.Aggregation() aggregation_pb.alias = self.alias - aggregation_pb.sum_ = query_pb2.AggregationQuery.Aggregation.Sum() - aggregation_pb.sum_.property.name = self.property_ref + aggregation_pb.sum = query_pb2.AggregationQuery.Aggregation.Sum() + aggregation_pb.sum.property.name = self.property_ref aggregation_pb.alias = self.alias return aggregation_pb diff --git a/tests/unit/test_aggregation.py b/tests/unit/test_aggregation.py index 8a1b4874..3f562866 100644 --- a/tests/unit/test_aggregation.py +++ b/tests/unit/test_aggregation.py @@ -45,8 +45,8 @@ def test_sum_aggregation_to_pb(): sum_aggregation = SumAggregation("person", alias="total") expected_aggregation_query_pb = query_pb2.AggregationQuery.Aggregation() - expected_aggregation_query_pb.sum_ = query_pb2.AggregationQuery.Aggregation.Sum() - expected_aggregation_query_pb.sum_.property.name = sum_aggregation.property_ref + expected_aggregation_query_pb.sum = query_pb2.AggregationQuery.Aggregation.Sum() + expected_aggregation_query_pb.sum.property.name = sum_aggregation.property_ref expected_aggregation_query_pb.alias = sum_aggregation.alias assert sum_aggregation._to_pb() == expected_aggregation_query_pb @@ -347,8 +347,8 @@ def test_iterator__build_protobuf_all_values(): expected_pb.aggregations.append(expected_count_pb) expected_sum_pb = query_pb2.AggregationQuery.Aggregation() - expected_sum_pb.sum_ = query_pb2.AggregationQuery.Aggregation.Sum() - expected_sum_pb.sum_.property.name = property_ref + expected_sum_pb.sum = query_pb2.AggregationQuery.Aggregation.Sum() + expected_sum_pb.sum.property.name = property_ref expected_pb.aggregations.append(expected_sum_pb) expected_avg_pb = query_pb2.AggregationQuery.Aggregation() From 493b56fc6456d9672fed3b29b048eae74b411b7f Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 18 Aug 2023 15:35:32 -0700 Subject: [PATCH 03/18] added missing test decorators --- tests/unit/test_aggregation.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_aggregation.py b/tests/unit/test_aggregation.py index 3f562866..8c155ff9 100644 --- a/tests/unit/test_aggregation.py +++ b/tests/unit/test_aggregation.py @@ -162,8 +162,8 @@ def test_pb_over_query_with_add_aggregations(client, database_id): assert pb.aggregations[2] == SumAggregation("person", alias="sum_person")._to_pb() assert pb.aggregations[3] == AvgAggregation("person", alias="avg_person")._to_pb() - -def test_pb_over_query_with_sum(client): +@pytest.mark.parametrize("database_id", [None, "somedb"], indirect=True) +def test_pb_over_query_with_sum(client, database_id): from google.cloud.datastore.query import _pb_from_query query = _make_query(client) @@ -176,7 +176,8 @@ def test_pb_over_query_with_sum(client): assert pb.aggregations[0] == SumAggregation("person", alias="total")._to_pb() -def test_pb_over_query_sum_with_add_aggregation(client): +@pytest.mark.parametrize("database_id", [None, "somedb"], indirect=True) +def test_pb_over_query_sum_with_add_aggregation(client, database_id): from google.cloud.datastore.query import _pb_from_query query = _make_query(client) @@ -189,7 +190,8 @@ def test_pb_over_query_sum_with_add_aggregation(client): assert pb.aggregations[0] == SumAggregation("person", alias="total")._to_pb() -def test_pb_over_query_with_avg(client): +@pytest.mark.parametrize("database_id", [None, "somedb"], indirect=True) +def test_pb_over_query_with_avg(client, database_id): from google.cloud.datastore.query import _pb_from_query query = _make_query(client) @@ -202,7 +204,8 @@ def test_pb_over_query_with_avg(client): assert pb.aggregations[0] == AvgAggregation("person", alias="total")._to_pb() -def test_pb_over_query_avf_with_add_aggregation(client): +@pytest.mark.parametrize("database_id", [None, "somedb"], indirect=True) +def test_pb_over_query_avf_with_add_aggregation(client, database_id): from google.cloud.datastore.query import _pb_from_query query = _make_query(client) From d1732594c25517d594493f3e081171bb1ad1b6db Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 18 Aug 2023 22:37:37 +0000 Subject: [PATCH 04/18] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- tests/unit/test_aggregation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_aggregation.py b/tests/unit/test_aggregation.py index 8c155ff9..30f58d68 100644 --- a/tests/unit/test_aggregation.py +++ b/tests/unit/test_aggregation.py @@ -162,6 +162,7 @@ def test_pb_over_query_with_add_aggregations(client, database_id): assert pb.aggregations[2] == SumAggregation("person", alias="sum_person")._to_pb() assert pb.aggregations[3] == AvgAggregation("person", alias="avg_person")._to_pb() + @pytest.mark.parametrize("database_id", [None, "somedb"], indirect=True) def test_pb_over_query_with_sum(client, database_id): from google.cloud.datastore.query import _pb_from_query From 27bea7bc3a979e4c8784364fe6028e7afd8770ef Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 25 Aug 2023 16:42:19 -0700 Subject: [PATCH 05/18] added missing decorators --- tests/system/test_aggregation_query.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/system/test_aggregation_query.py b/tests/system/test_aggregation_query.py index 76a1afa0..3cb67ece 100644 --- a/tests/system/test_aggregation_query.py +++ b/tests/system/test_aggregation_query.py @@ -97,7 +97,8 @@ def test_aggregation_query_with_alias( assert r.value > 0 -def test_sum_query_default(aggregation_query_client, nested_query): +@pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) +def test_sum_query_default(aggregation_query_client, nested_query, database_id): query = nested_query aggregation_query = aggregation_query_client.aggregation_query(query) @@ -109,7 +110,8 @@ def test_sum_query_default(aggregation_query_client, nested_query): assert r.value == 8 -def test_sum_query_with_alias(aggregation_query_client, nested_query): +@pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) +def test_sum_query_with_alias(aggregation_query_client, nested_query, database_id): query = nested_query aggregation_query = aggregation_query_client.aggregation_query(query) @@ -121,7 +123,8 @@ def test_sum_query_with_alias(aggregation_query_client, nested_query): assert r.value > 0 -def test_avg_query_default(aggregation_query_client, nested_query): +@pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) +def test_avg_query_default(aggregation_query_client, nested_query, database_id): query = nested_query aggregation_query = aggregation_query_client.aggregation_query(query) @@ -133,7 +136,8 @@ def test_avg_query_default(aggregation_query_client, nested_query): assert r.value == 8 -def test_avg_query_with_alias(aggregation_query_client, nested_query): +@pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) +def test_avg_query_with_alias(aggregation_query_client, nested_query, database_id): query = nested_query aggregation_query = aggregation_query_client.aggregation_query(query) From c2b624e2a7d2e97e48052893fe4d5a9f22707168 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 25 Aug 2023 16:53:41 -0700 Subject: [PATCH 06/18] added indices --- tests/system/index.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/system/index.yaml b/tests/system/index.yaml index f9cc2a5b..68c9cae9 100644 --- a/tests/system/index.yaml +++ b/tests/system/index.yaml @@ -45,3 +45,21 @@ indexes: properties: - name: family - name: appearances + + - kind: Character + ancestor: yes + properties: + - name: person + +- kind: Character + ancestor: yes + properties: + - name: appearances + - name: person + +- kind: Character + ancestor: yes + properties: + - name: family + - name: appearances + - name: person From 197e72ec957fd443246cbef3349b5cc2de1fba2b Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 25 Aug 2023 17:16:47 -0700 Subject: [PATCH 07/18] fixed index formatting issue --- tests/system/index.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/system/index.yaml b/tests/system/index.yaml index 68c9cae9..53f788db 100644 --- a/tests/system/index.yaml +++ b/tests/system/index.yaml @@ -39,21 +39,21 @@ indexes: - name: family - name: appearances - - kind: Character ancestor: yes properties: - name: family - name: appearances - - kind: Character - ancestor: yes - properties: - - name: person +- kind: Character + ancestor: yes + properties: + - name: person - kind: Character ancestor: yes properties: + - name: family - name: appearances - name: person From ab6d91bb19b806b7537ca759542f6fdb2e53e613 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 25 Aug 2023 17:23:37 -0700 Subject: [PATCH 08/18] changed test aggregation field name --- tests/system/index.yaml | 18 ------------------ tests/unit/test_aggregation.py | 33 ++++++++++++++++----------------- 2 files changed, 16 insertions(+), 35 deletions(-) diff --git a/tests/system/index.yaml b/tests/system/index.yaml index 53f788db..1f27c246 100644 --- a/tests/system/index.yaml +++ b/tests/system/index.yaml @@ -45,21 +45,3 @@ indexes: - name: family - name: appearances -- kind: Character - ancestor: yes - properties: - - name: person - -- kind: Character - ancestor: yes - properties: - - name: family - - name: appearances - - name: person - -- kind: Character - ancestor: yes - properties: - - name: family - - name: appearances - - name: person diff --git a/tests/unit/test_aggregation.py b/tests/unit/test_aggregation.py index 30f58d68..0110a113 100644 --- a/tests/unit/test_aggregation.py +++ b/tests/unit/test_aggregation.py @@ -42,7 +42,7 @@ def test_count_aggregation_to_pb(): def test_sum_aggregation_to_pb(): from google.cloud.datastore_v1.types import query as query_pb2 - sum_aggregation = SumAggregation("person", alias="total") + sum_aggregation = SumAggregation("appearances", alias="total") expected_aggregation_query_pb = query_pb2.AggregationQuery.Aggregation() expected_aggregation_query_pb.sum = query_pb2.AggregationQuery.Aggregation.Sum() @@ -54,7 +54,7 @@ def test_sum_aggregation_to_pb(): def test_avg_aggregation_to_pb(): from google.cloud.datastore_v1.types import query as query_pb2 - avg_aggregation = AvgAggregation("person", alias="total") + avg_aggregation = AvgAggregation("appearances", alias="total") expected_aggregation_query_pb = query_pb2.AggregationQuery.Aggregation() expected_aggregation_query_pb.avg = query_pb2.AggregationQuery.Aggregation.Avg() @@ -146,8 +146,8 @@ def test_pb_over_query_with_add_aggregations(client, database_id): aggregations = [ CountAggregation(alias="total"), CountAggregation(alias="all"), - SumAggregation("person", alias="sum_person"), - AvgAggregation("person", alias="avg_person"), + SumAggregation("appearances", alias="sum_appearances"), + AvgAggregation("appearances", alias="avg_appearances"), ] query = _make_query(client) @@ -159,9 +159,8 @@ def test_pb_over_query_with_add_aggregations(client, database_id): assert len(pb.aggregations) == 4 assert pb.aggregations[0] == CountAggregation(alias="total")._to_pb() assert pb.aggregations[1] == CountAggregation(alias="all")._to_pb() - assert pb.aggregations[2] == SumAggregation("person", alias="sum_person")._to_pb() - assert pb.aggregations[3] == AvgAggregation("person", alias="avg_person")._to_pb() - + assert pb.aggregations[2] == SumAggregation("appearances", alias="sum_appearances")._to_pb() + assert pb.aggregations[3] == AvgAggregation("appearances", alias="avg_appearances")._to_pb() @pytest.mark.parametrize("database_id", [None, "somedb"], indirect=True) def test_pb_over_query_with_sum(client, database_id): @@ -170,11 +169,11 @@ def test_pb_over_query_with_sum(client, database_id): query = _make_query(client) aggregation_query = _make_aggregation_query(client=client, query=query) - aggregation_query.sum("person", alias="total") + aggregation_query.sum("appearances", alias="total") pb = aggregation_query._to_pb() assert pb.nested_query == _pb_from_query(query) assert len(pb.aggregations) == 1 - assert pb.aggregations[0] == SumAggregation("person", alias="total")._to_pb() + assert pb.aggregations[0] == SumAggregation("appearances", alias="total")._to_pb() @pytest.mark.parametrize("database_id", [None, "somedb"], indirect=True) @@ -184,11 +183,11 @@ def test_pb_over_query_sum_with_add_aggregation(client, database_id): query = _make_query(client) aggregation_query = _make_aggregation_query(client=client, query=query) - aggregation_query.add_aggregation(SumAggregation("person", alias="total")) + aggregation_query.add_aggregation(SumAggregation("appearances", alias="total")) pb = aggregation_query._to_pb() assert pb.nested_query == _pb_from_query(query) assert len(pb.aggregations) == 1 - assert pb.aggregations[0] == SumAggregation("person", alias="total")._to_pb() + assert pb.aggregations[0] == SumAggregation("appearances", alias="total")._to_pb() @pytest.mark.parametrize("database_id", [None, "somedb"], indirect=True) @@ -198,25 +197,25 @@ def test_pb_over_query_with_avg(client, database_id): query = _make_query(client) aggregation_query = _make_aggregation_query(client=client, query=query) - aggregation_query.avg("person", alias="total") + aggregation_query.avg("appearances", alias="avg") pb = aggregation_query._to_pb() assert pb.nested_query == _pb_from_query(query) assert len(pb.aggregations) == 1 - assert pb.aggregations[0] == AvgAggregation("person", alias="total")._to_pb() + assert pb.aggregations[0] == AvgAggregation("appearances", alias="avg")._to_pb() @pytest.mark.parametrize("database_id", [None, "somedb"], indirect=True) -def test_pb_over_query_avf_with_add_aggregation(client, database_id): +def test_pb_over_query_avg_with_add_aggregation(client, database_id): from google.cloud.datastore.query import _pb_from_query query = _make_query(client) aggregation_query = _make_aggregation_query(client=client, query=query) - aggregation_query.add_aggregation(AvgAggregation("person", alias="total")) + aggregation_query.add_aggregation(AvgAggregation("appearances", alias="avg")) pb = aggregation_query._to_pb() assert pb.nested_query == _pb_from_query(query) assert len(pb.aggregations) == 1 - assert pb.aggregations[0] == AvgAggregation("person", alias="total")._to_pb() + assert pb.aggregations[0] == AvgAggregation("appearances", alias="avg")._to_pb() @pytest.mark.parametrize("database_id", [None, "somedb"], indirect=True) @@ -332,7 +331,7 @@ def test_iterator__build_protobuf_all_values(): query = _make_query(client) alias = "total" limit = 2 - property_ref = "person" + property_ref = "appearances" aggregation_query = AggregationQuery(client=client, query=query) aggregation_query.count(alias) aggregation_query.sum(property_ref) From 25d35fd76b2674f16d06ca9ffb5068561e87c8d9 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 25 Aug 2023 17:24:37 -0700 Subject: [PATCH 09/18] ran blacken --- tests/unit/test_aggregation.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_aggregation.py b/tests/unit/test_aggregation.py index 0110a113..123433bc 100644 --- a/tests/unit/test_aggregation.py +++ b/tests/unit/test_aggregation.py @@ -159,8 +159,15 @@ def test_pb_over_query_with_add_aggregations(client, database_id): assert len(pb.aggregations) == 4 assert pb.aggregations[0] == CountAggregation(alias="total")._to_pb() assert pb.aggregations[1] == CountAggregation(alias="all")._to_pb() - assert pb.aggregations[2] == SumAggregation("appearances", alias="sum_appearances")._to_pb() - assert pb.aggregations[3] == AvgAggregation("appearances", alias="avg_appearances")._to_pb() + assert ( + pb.aggregations[2] + == SumAggregation("appearances", alias="sum_appearances")._to_pb() + ) + assert ( + pb.aggregations[3] + == AvgAggregation("appearances", alias="avg_appearances")._to_pb() + ) + @pytest.mark.parametrize("database_id", [None, "somedb"], indirect=True) def test_pb_over_query_with_sum(client, database_id): From 68e44aef426771574c702b00ab468d884b6aaa55 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 29 Aug 2023 15:28:09 -0700 Subject: [PATCH 10/18] replaced person with appearances in tests --- tests/system/test_aggregation_query.py | 42 +++++++++++++------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/system/test_aggregation_query.py b/tests/system/test_aggregation_query.py index 3cb67ece..e3e5bfb3 100644 --- a/tests/system/test_aggregation_query.py +++ b/tests/system/test_aggregation_query.py @@ -102,7 +102,7 @@ def test_sum_query_default(aggregation_query_client, nested_query, database_id): query = nested_query aggregation_query = aggregation_query_client.aggregation_query(query) - aggregation_query.sum("person") + aggregation_query.sum("appearances") result = _do_fetch(aggregation_query) assert len(result) == 1 for r in result[0]: @@ -115,11 +115,11 @@ def test_sum_query_with_alias(aggregation_query_client, nested_query, database_i query = nested_query aggregation_query = aggregation_query_client.aggregation_query(query) - aggregation_query.sum("person", alias="sum_person") + aggregation_query.sum("appearances", alias="sum_appearances") result = _do_fetch(aggregation_query) assert len(result) == 1 for r in result[0]: - assert r.alias == "sum_person" + assert r.alias == "sum_appearances" assert r.value > 0 @@ -128,7 +128,7 @@ def test_avg_query_default(aggregation_query_client, nested_query, database_id): query = nested_query aggregation_query = aggregation_query_client.aggregation_query(query) - aggregation_query.avg("person") + aggregation_query.avg("appearances") result = _do_fetch(aggregation_query) assert len(result) == 1 for r in result[0]: @@ -141,11 +141,11 @@ def test_avg_query_with_alias(aggregation_query_client, nested_query, database_i query = nested_query aggregation_query = aggregation_query_client.aggregation_query(query) - aggregation_query.avg("person", alias="avg_person") + aggregation_query.avg("appearances", alias="avg_appearances") result = _do_fetch(aggregation_query) assert len(result) == 1 for r in result[0]: - assert r.alias == "avg_person" + assert r.alias == "avg_appearances" assert r.value > 0 @@ -181,12 +181,12 @@ def test_aggregation_query_multiple_aggregations( aggregation_query = aggregation_query_client.aggregation_query(query) aggregation_query.count(alias="total") aggregation_query.count(alias="all") - aggregation_query.sum("person", alias="sum_person") - aggregation_query.avg("person", alias="avg_person") + aggregation_query.sum("appearances", alias="sum_appearances") + aggregation_query.avg("appearances", alias="avg_appearances") result = _do_fetch(aggregation_query) assert len(result) == 1 for r in result[0]: - assert r.alias in ["all", "total", "sum_person", "avg_person"] + assert r.alias in ["all", "total", "sum_appearances", "avg_appearances"] assert r.value > 0 @@ -204,16 +204,16 @@ def test_aggregation_query_add_aggregation( count_aggregation = CountAggregation(alias="total") aggregation_query.add_aggregation(count_aggregation) - sum_aggregation = SumAggregation("person", alias="sum_person") + sum_aggregation = SumAggregation("appearances", alias="sum_appearances") aggregation_query.add_aggregation(sum_aggregation) - avg_aggregation = AvgAggregation("person", alias="avg_person") + avg_aggregation = AvgAggregation("appearances", alias="avg_appearances") aggregation_query.add_aggregation(avg_aggregation) result = _do_fetch(aggregation_query) assert len(result) == 1 for r in result[0]: - assert r.alias in ["total", "sum_person", "avg_person"] + assert r.alias in ["total", "sum_appearances", "avg_appearances"] assert r.value > 0 @@ -232,15 +232,15 @@ def test_aggregation_query_add_aggregations( aggregation_query = aggregation_query_client.aggregation_query(query) count_aggregation_1 = CountAggregation(alias="total") count_aggregation_2 = CountAggregation(alias="all") - sum_aggregation = SumAggregation("person", alias="sum_person") - avg_aggregation = AvgAggregation("person", alias="avg_person") + sum_aggregation = SumAggregation("appearances", alias="sum_appearances") + avg_aggregation = AvgAggregation("appearances", alias="avg_appearances") aggregation_query.add_aggregations( [count_aggregation_1, count_aggregation_2, sum_aggregation, avg_aggregation] ) result = _do_fetch(aggregation_query) assert len(result) == 1 for r in result[0]: - assert r.alias in ["total", "all", "sum_person", "avg_person"] + assert r.alias in ["total", "all", "sum_appearances", "avg_appearances"] assert r.value > 0 @@ -289,13 +289,13 @@ def test_aggregation_query_with_nested_query_filtered( aggregation_query = aggregation_query_client.aggregation_query(query) aggregation_query.count(alias="total") - aggregation_query.sum("person", alias="sum_person") - aggregation_query.avg("person", alias="avg_person") + aggregation_query.sum("appearances", alias="sum_appearances") + aggregation_query.avg("appearances", alias="avg_appearances") result = _do_fetch(aggregation_query) assert len(result) == 1 for r in result[0]: - assert r.alias in ["total", "sum_person", "avg_person"] + assert r.alias in ["total", "sum_appearances", "avg_appearances"] assert r.value == 6 @@ -316,11 +316,11 @@ def test_aggregation_query_with_nested_query_multiple_filters( aggregation_query = aggregation_query_client.aggregation_query(query) aggregation_query.count(alias="total") - aggregation_query.sum("person", alias="sum_person") - aggregation_query.avg("person", alias="avg_person") + aggregation_query.sum("appearances", alias="sum_appearances") + aggregation_query.avg("appearances", alias="avg_appearances") result = _do_fetch(aggregation_query) assert len(result) == 1 for r in result[0]: - assert r.alias in ["total", "sum_person", "avg_person"] + assert r.alias in ["total", "sum_appearances", "avg_appearances"] assert r.value == 4 From bbdc2c24882643071936a284fed1b9f61acd0abb Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 29 Aug 2023 16:28:49 -0700 Subject: [PATCH 11/18] added tests for sum and avg limits --- tests/system/test_aggregation_query.py | 37 +++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/tests/system/test_aggregation_query.py b/tests/system/test_aggregation_query.py index e3e5bfb3..652b0449 100644 --- a/tests/system/test_aggregation_query.py +++ b/tests/system/test_aggregation_query.py @@ -150,7 +150,7 @@ def test_avg_query_with_alias(aggregation_query_client, nested_query, database_i @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) -def test_aggregation_query_with_limit( +def test_count_query_with_limit( aggregation_query_client, nested_query, database_id ): query = nested_query @@ -171,6 +171,41 @@ def test_aggregation_query_with_limit( assert r.alias == "total_up_to" assert r.value == 2 +@pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) +def test_sum_query_with_limit( + aggregation_query_client, nested_query, database_id +): + query = nested_query + + aggregation_query = aggregation_query_client.aggregation_query(query) + aggregation_query.sum("appearances", alias="sum_limited") + limit = 2 + result = _do_fetch(aggregation_query, limit=limit) # count with limit = 2 + assert len(result) == 1 + assert len(result[0]) == 1 + r = result[0][0] + assert r.alias == "sum_limited" + expected = sum(c["appearances"] for c in populate_datastore.CHARACTERS[:limit]) + assert r.value == expected + + +@pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) +def test_avg_query_with_limit( + aggregation_query_client, nested_query, database_id +): + query = nested_query + + aggregation_query = aggregation_query_client.aggregation_query(query) + aggregation_query.avg("appearances", alias="avg_limited") + limit = 2 + result = _do_fetch(aggregation_query, limit=limit) # count with limit = 2 + assert len(result) == 1 + assert len(result[0]) == 1 + r = result[0][0] + assert r.alias == "avg_limited" + expected = sum(c["appearances"] for c in populate_datastore.CHARACTERS[:limit]) / limit + assert r.value == expected + @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) def test_aggregation_query_multiple_aggregations( From 987e57978e26a62d42158d42c0f9a6a30dddc62e Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 29 Aug 2023 16:29:15 -0700 Subject: [PATCH 12/18] aggregation results support double values --- google/cloud/datastore/aggregation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/google/cloud/datastore/aggregation.py b/google/cloud/datastore/aggregation.py index 360d055a..e2198270 100644 --- a/google/cloud/datastore/aggregation.py +++ b/google/cloud/datastore/aggregation.py @@ -517,5 +517,8 @@ def _item_to_aggregation_result(iterator, pb): :rtype: :class:`google.cloud.datastore.aggregation.AggregationResult` :returns: The list of AggregationResults """ - results = [AggregationResult(alias=k, value=pb[k].integer_value) for k in pb.keys()] + results = [ + AggregationResult(alias=k, value=pb[k].integer_value or pb[k].double_value) + for k in pb.keys() + ] return results From e5606624891fd1909f1d8802bf5e79186c830b83 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 29 Aug 2023 16:29:25 -0700 Subject: [PATCH 13/18] refactored aggregation tests --- tests/system/test_aggregation_query.py | 116 +++++++++++++++---------- 1 file changed, 72 insertions(+), 44 deletions(-) diff --git a/tests/system/test_aggregation_query.py b/tests/system/test_aggregation_query.py index 652b0449..c71009c4 100644 --- a/tests/system/test_aggregation_query.py +++ b/tests/system/test_aggregation_query.py @@ -70,20 +70,22 @@ def nested_query(aggregation_query_client, ancestor_key): @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) -def test_aggregation_query_default(aggregation_query_client, nested_query, database_id): +def test_count_query_default(aggregation_query_client, nested_query, database_id): query = nested_query aggregation_query = aggregation_query_client.aggregation_query(query) aggregation_query.count() result = _do_fetch(aggregation_query) assert len(result) == 1 - for r in result[0]: - assert r.alias == "property_1" - assert r.value == 8 + assert len(result[0]) == 1 + r = result[0][0] + assert r.alias == "property_1" + expected_count = len(populate_datastore.CHARACTERS) + assert r.value == expected_count @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) -def test_aggregation_query_with_alias( +def test_count_query_with_alias( aggregation_query_client, nested_query, database_id ): query = nested_query @@ -92,9 +94,11 @@ def test_aggregation_query_with_alias( aggregation_query.count(alias="total") result = _do_fetch(aggregation_query) assert len(result) == 1 - for r in result[0]: - assert r.alias == "total" - assert r.value > 0 + assert len(result[0]) == 1 + r = result[0][0] + assert r.alias == "total" + expected_count = len(populate_datastore.CHARACTERS) + assert r.value == expected_count @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) @@ -105,9 +109,11 @@ def test_sum_query_default(aggregation_query_client, nested_query, database_id): aggregation_query.sum("appearances") result = _do_fetch(aggregation_query) assert len(result) == 1 - for r in result[0]: - assert r.alias == "property_1" - assert r.value == 8 + assert len(result[0]) == 1 + r = result[0][0] + assert r.alias == "property_1" + expected_sum = sum(c["appearances"] for c in populate_datastore.CHARACTERS) + assert r.value == expected_sum @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) @@ -118,9 +124,11 @@ def test_sum_query_with_alias(aggregation_query_client, nested_query, database_i aggregation_query.sum("appearances", alias="sum_appearances") result = _do_fetch(aggregation_query) assert len(result) == 1 - for r in result[0]: - assert r.alias == "sum_appearances" - assert r.value > 0 + assert len(result[0]) == 1 + r = result[0][0] + assert r.alias == "sum_appearances" + expected_sum = sum(c["appearances"] for c in populate_datastore.CHARACTERS) + assert r.value == expected_sum @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) @@ -131,9 +139,11 @@ def test_avg_query_default(aggregation_query_client, nested_query, database_id): aggregation_query.avg("appearances") result = _do_fetch(aggregation_query) assert len(result) == 1 - for r in result[0]: - assert r.alias == "property_1" - assert r.value == 8 + assert len(result[0]) == 1 + r = result[0][0] + assert r.alias == "property_1" + expected_avg = sum(c["appearances"] for c in populate_datastore.CHARACTERS) / len(populate_datastore.CHARACTERS) + assert r.value == expected_avg @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) @@ -144,9 +154,11 @@ def test_avg_query_with_alias(aggregation_query_client, nested_query, database_i aggregation_query.avg("appearances", alias="avg_appearances") result = _do_fetch(aggregation_query) assert len(result) == 1 - for r in result[0]: - assert r.alias == "avg_appearances" - assert r.value > 0 + assert len(result[0]) == 1 + r = result[0][0] + assert r.alias == "avg_appearances" + expected_avg = sum(c["appearances"] for c in populate_datastore.CHARACTERS) / len(populate_datastore.CHARACTERS) + assert r.value == expected_avg @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) @@ -159,17 +171,21 @@ def test_count_query_with_limit( aggregation_query.count(alias="total") result = _do_fetch(aggregation_query) # count without limit assert len(result) == 1 - for r in result[0]: - assert r.alias == "total" - assert r.value == 8 + assert len(result[0]) == 1 + r = result[0][0] + assert r.alias == "total" + expected_count = len(populate_datastore.CHARACTERS) + assert r.value == expected_count aggregation_query = aggregation_query_client.aggregation_query(query) aggregation_query.count(alias="total_up_to") - result = _do_fetch(aggregation_query, limit=2) # count with limit = 2 + limit = 2 + result = _do_fetch(aggregation_query, limit=limit) # count with limit = 2 assert len(result) == 1 - for r in result[0]: - assert r.alias == "total_up_to" - assert r.value == 2 + assert len(result[0]) == 1 + r = result[0][0] + assert r.alias == "total_up_to" + assert r.value == limit @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) def test_sum_query_with_limit( @@ -220,9 +236,12 @@ def test_aggregation_query_multiple_aggregations( aggregation_query.avg("appearances", alias="avg_appearances") result = _do_fetch(aggregation_query) assert len(result) == 1 - for r in result[0]: - assert r.alias in ["all", "total", "sum_appearances", "avg_appearances"] - assert r.value > 0 + assert len(result[0]) == 4 + result_dict = {r.alias: r for r in result[0]} + assert result_dict["total"].value == len(populate_datastore.CHARACTERS) + assert result_dict["all"].value == len(populate_datastore.CHARACTERS) + assert result_dict["sum_appearances"].value == sum(c["appearances"] for c in populate_datastore.CHARACTERS) + assert result_dict["avg_appearances"].value == sum(c["appearances"] for c in populate_datastore.CHARACTERS) / len(populate_datastore.CHARACTERS) @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) @@ -247,9 +266,11 @@ def test_aggregation_query_add_aggregation( result = _do_fetch(aggregation_query) assert len(result) == 1 - for r in result[0]: - assert r.alias in ["total", "sum_appearances", "avg_appearances"] - assert r.value > 0 + assert len(result[0]) == 3 + result_dict = {r.alias: r for r in result[0]} + assert result_dict["total"].value == len(populate_datastore.CHARACTERS) + assert result_dict["sum_appearances"].value == sum(c["appearances"] for c in populate_datastore.CHARACTERS) + assert result_dict["avg_appearances"].value == sum(c["appearances"] for c in populate_datastore.CHARACTERS) / len(populate_datastore.CHARACTERS) @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) @@ -274,9 +295,12 @@ def test_aggregation_query_add_aggregations( ) result = _do_fetch(aggregation_query) assert len(result) == 1 - for r in result[0]: - assert r.alias in ["total", "all", "sum_appearances", "avg_appearances"] - assert r.value > 0 + assert len(result[0]) == 4 + result_dict = {r.alias: r for r in result[0]} + assert result_dict["total"].value == len(populate_datastore.CHARACTERS) + assert result_dict["all"].value == len(populate_datastore.CHARACTERS) + assert result_dict["sum_appearances"].value == sum(c["appearances"] for c in populate_datastore.CHARACTERS) + assert result_dict["avg_appearances"].value == sum(c["appearances"] for c in populate_datastore.CHARACTERS) / len(populate_datastore.CHARACTERS) @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) @@ -328,10 +352,12 @@ def test_aggregation_query_with_nested_query_filtered( aggregation_query.avg("appearances", alias="avg_appearances") result = _do_fetch(aggregation_query) assert len(result) == 1 - - for r in result[0]: - assert r.alias in ["total", "sum_appearances", "avg_appearances"] - assert r.value == 6 + assert len(result[0]) == 3 + result_dict = {r.alias: r for r in result[0]} + assert result_dict["total"].value == expected_matches + expected_sum = sum(c["appearances"] for c in populate_datastore.CHARACTERS if c["appearances"] >= 20) + assert result_dict["sum_appearances"].value == expected_sum + assert result_dict["avg_appearances"].value == expected_sum / expected_matches @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) @@ -355,7 +381,9 @@ def test_aggregation_query_with_nested_query_multiple_filters( aggregation_query.avg("appearances", alias="avg_appearances") result = _do_fetch(aggregation_query) assert len(result) == 1 - - for r in result[0]: - assert r.alias in ["total", "sum_appearances", "avg_appearances"] - assert r.value == 4 + assert len(result[0]) == 3 + result_dict = {r.alias: r for r in result[0]} + assert result_dict["total"].value == expected_matches + expected_sum = sum(c["appearances"] for c in populate_datastore.CHARACTERS if c["appearances"] >= 26 and "Stark" in c["family"]) + assert result_dict["sum_appearances"].value == expected_sum + assert result_dict["avg_appearances"].value == expected_sum / expected_matches From bd52f4b9738c7ac395e43d5921cd5847104682a6 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 29 Aug 2023 23:31:34 +0000 Subject: [PATCH 14/18] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- tests/system/test_aggregation_query.py | 65 +++++++++++++++++--------- 1 file changed, 42 insertions(+), 23 deletions(-) diff --git a/tests/system/test_aggregation_query.py b/tests/system/test_aggregation_query.py index c71009c4..abce31e5 100644 --- a/tests/system/test_aggregation_query.py +++ b/tests/system/test_aggregation_query.py @@ -85,9 +85,7 @@ def test_count_query_default(aggregation_query_client, nested_query, database_id @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) -def test_count_query_with_alias( - aggregation_query_client, nested_query, database_id -): +def test_count_query_with_alias(aggregation_query_client, nested_query, database_id): query = nested_query aggregation_query = aggregation_query_client.aggregation_query(query) @@ -142,7 +140,9 @@ def test_avg_query_default(aggregation_query_client, nested_query, database_id): assert len(result[0]) == 1 r = result[0][0] assert r.alias == "property_1" - expected_avg = sum(c["appearances"] for c in populate_datastore.CHARACTERS) / len(populate_datastore.CHARACTERS) + expected_avg = sum(c["appearances"] for c in populate_datastore.CHARACTERS) / len( + populate_datastore.CHARACTERS + ) assert r.value == expected_avg @@ -157,14 +157,14 @@ def test_avg_query_with_alias(aggregation_query_client, nested_query, database_i assert len(result[0]) == 1 r = result[0][0] assert r.alias == "avg_appearances" - expected_avg = sum(c["appearances"] for c in populate_datastore.CHARACTERS) / len(populate_datastore.CHARACTERS) + expected_avg = sum(c["appearances"] for c in populate_datastore.CHARACTERS) / len( + populate_datastore.CHARACTERS + ) assert r.value == expected_avg @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) -def test_count_query_with_limit( - aggregation_query_client, nested_query, database_id -): +def test_count_query_with_limit(aggregation_query_client, nested_query, database_id): query = nested_query aggregation_query = aggregation_query_client.aggregation_query(query) @@ -187,10 +187,9 @@ def test_count_query_with_limit( assert r.alias == "total_up_to" assert r.value == limit + @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) -def test_sum_query_with_limit( - aggregation_query_client, nested_query, database_id -): +def test_sum_query_with_limit(aggregation_query_client, nested_query, database_id): query = nested_query aggregation_query = aggregation_query_client.aggregation_query(query) @@ -206,9 +205,7 @@ def test_sum_query_with_limit( @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) -def test_avg_query_with_limit( - aggregation_query_client, nested_query, database_id -): +def test_avg_query_with_limit(aggregation_query_client, nested_query, database_id): query = nested_query aggregation_query = aggregation_query_client.aggregation_query(query) @@ -219,7 +216,9 @@ def test_avg_query_with_limit( assert len(result[0]) == 1 r = result[0][0] assert r.alias == "avg_limited" - expected = sum(c["appearances"] for c in populate_datastore.CHARACTERS[:limit]) / limit + expected = ( + sum(c["appearances"] for c in populate_datastore.CHARACTERS[:limit]) / limit + ) assert r.value == expected @@ -240,8 +239,12 @@ def test_aggregation_query_multiple_aggregations( result_dict = {r.alias: r for r in result[0]} assert result_dict["total"].value == len(populate_datastore.CHARACTERS) assert result_dict["all"].value == len(populate_datastore.CHARACTERS) - assert result_dict["sum_appearances"].value == sum(c["appearances"] for c in populate_datastore.CHARACTERS) - assert result_dict["avg_appearances"].value == sum(c["appearances"] for c in populate_datastore.CHARACTERS) / len(populate_datastore.CHARACTERS) + assert result_dict["sum_appearances"].value == sum( + c["appearances"] for c in populate_datastore.CHARACTERS + ) + assert result_dict["avg_appearances"].value == sum( + c["appearances"] for c in populate_datastore.CHARACTERS + ) / len(populate_datastore.CHARACTERS) @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) @@ -269,8 +272,12 @@ def test_aggregation_query_add_aggregation( assert len(result[0]) == 3 result_dict = {r.alias: r for r in result[0]} assert result_dict["total"].value == len(populate_datastore.CHARACTERS) - assert result_dict["sum_appearances"].value == sum(c["appearances"] for c in populate_datastore.CHARACTERS) - assert result_dict["avg_appearances"].value == sum(c["appearances"] for c in populate_datastore.CHARACTERS) / len(populate_datastore.CHARACTERS) + assert result_dict["sum_appearances"].value == sum( + c["appearances"] for c in populate_datastore.CHARACTERS + ) + assert result_dict["avg_appearances"].value == sum( + c["appearances"] for c in populate_datastore.CHARACTERS + ) / len(populate_datastore.CHARACTERS) @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) @@ -299,8 +306,12 @@ def test_aggregation_query_add_aggregations( result_dict = {r.alias: r for r in result[0]} assert result_dict["total"].value == len(populate_datastore.CHARACTERS) assert result_dict["all"].value == len(populate_datastore.CHARACTERS) - assert result_dict["sum_appearances"].value == sum(c["appearances"] for c in populate_datastore.CHARACTERS) - assert result_dict["avg_appearances"].value == sum(c["appearances"] for c in populate_datastore.CHARACTERS) / len(populate_datastore.CHARACTERS) + assert result_dict["sum_appearances"].value == sum( + c["appearances"] for c in populate_datastore.CHARACTERS + ) + assert result_dict["avg_appearances"].value == sum( + c["appearances"] for c in populate_datastore.CHARACTERS + ) / len(populate_datastore.CHARACTERS) @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) @@ -355,7 +366,11 @@ def test_aggregation_query_with_nested_query_filtered( assert len(result[0]) == 3 result_dict = {r.alias: r for r in result[0]} assert result_dict["total"].value == expected_matches - expected_sum = sum(c["appearances"] for c in populate_datastore.CHARACTERS if c["appearances"] >= 20) + expected_sum = sum( + c["appearances"] + for c in populate_datastore.CHARACTERS + if c["appearances"] >= 20 + ) assert result_dict["sum_appearances"].value == expected_sum assert result_dict["avg_appearances"].value == expected_sum / expected_matches @@ -384,6 +399,10 @@ def test_aggregation_query_with_nested_query_multiple_filters( assert len(result[0]) == 3 result_dict = {r.alias: r for r in result[0]} assert result_dict["total"].value == expected_matches - expected_sum = sum(c["appearances"] for c in populate_datastore.CHARACTERS if c["appearances"] >= 26 and "Stark" in c["family"]) + expected_sum = sum( + c["appearances"] + for c in populate_datastore.CHARACTERS + if c["appearances"] >= 26 and "Stark" in c["family"] + ) assert result_dict["sum_appearances"].value == expected_sum assert result_dict["avg_appearances"].value == expected_sum / expected_matches From 3a6030d075209cf51409eccbddb1ec8132ee9fef Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 30 Aug 2023 11:31:41 -0700 Subject: [PATCH 15/18] added additional sum/avg tests --- tests/system/test_aggregation_query.py | 71 ++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/tests/system/test_aggregation_query.py b/tests/system/test_aggregation_query.py index abce31e5..05e7dadc 100644 --- a/tests/system/test_aggregation_query.py +++ b/tests/system/test_aggregation_query.py @@ -187,6 +187,15 @@ def test_count_query_with_limit(aggregation_query_client, nested_query, database assert r.alias == "total_up_to" assert r.value == limit + aggregation_query = aggregation_query_client.aggregation_query(query) + aggregation_query.count(alias="total_high_limit") + limit = 2 + result = _do_fetch(aggregation_query, limit=expected_count*2) # count with limit > total + assert len(result) == 1 + assert len(result[0]) == 1 + r = result[0][0] + assert r.alias == "total_high_limit" + assert r.value == expected_count @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) def test_sum_query_with_limit(aggregation_query_client, nested_query, database_id): @@ -203,6 +212,16 @@ def test_sum_query_with_limit(aggregation_query_client, nested_query, database_i expected = sum(c["appearances"] for c in populate_datastore.CHARACTERS[:limit]) assert r.value == expected + aggregation_query = aggregation_query_client.aggregation_query(query) + aggregation_query.sum("appearances", alias="sum_high_limit") + num_characters = len(populate_datastore.CHARACTERS) + result = _do_fetch(aggregation_query, limit=num_characters*2) # count with limit > total + assert len(result) == 1 + assert len(result[0]) == 1 + r = result[0][0] + assert r.alias == "sum_high_limit" + assert r.value == sum(c["appearances"] for c in populate_datastore.CHARACTERS) + @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) def test_avg_query_with_limit(aggregation_query_client, nested_query, database_id): @@ -221,6 +240,58 @@ def test_avg_query_with_limit(aggregation_query_client, nested_query, database_i ) assert r.value == expected + aggregation_query = aggregation_query_client.aggregation_query(query) + aggregation_query.avg("appearances", alias="avg_high_limit") + num_characters = len(populate_datastore.CHARACTERS) + result = _do_fetch(aggregation_query, limit=num_characters*2) # count with limit > total + assert len(result) == 1 + assert len(result[0]) == 1 + r = result[0][0] + assert r.alias == "avg_high_limit" + assert r.value == sum(c["appearances"] for c in populate_datastore.CHARACTERS) / num_characters + + +@pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) +def test_count_query_empty(aggregation_query_client, nested_query, database_id): + query = nested_query + query.add_filter("name", "=", "nonexistent") + aggregation_query = aggregation_query_client.aggregation_query(query) + aggregation_query.count(alias="total") + result = _do_fetch(aggregation_query) + assert len(result) == 1 + assert len(result[0]) == 1 + r = result[0][0] + assert r.alias == "total" + assert r.value == 0 + + +@pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) +def test_sum_query_empty(aggregation_query_client, nested_query, database_id): + query = nested_query + query.add_filter("family", "=", "nonexistent") + aggregation_query = aggregation_query_client.aggregation_query(query) + aggregation_query.sum("appearances", alias="sum") + result = _do_fetch(aggregation_query) + assert len(result) == 1 + assert len(result[0]) == 1 + r = result[0][0] + assert r.alias == "sum" + assert r.value == 0 + + +@pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) +def test_avg_query_empty(aggregation_query_client, nested_query, database_id): + query = nested_query + query.add_filter("family", "=", "nonexistent") + aggregation_query = aggregation_query_client.aggregation_query(query) + aggregation_query.avg("appearances", alias="avg") + result = _do_fetch(aggregation_query) + assert len(result) == 1 + assert len(result[0]) == 1 + r = result[0][0] + assert r.alias == "avg" + assert r.value == 0 + @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) def test_aggregation_query_multiple_aggregations( From c09db3ff61cb70434e94fe4a66b2a7ee676c212a Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 30 Aug 2023 12:46:09 -0700 Subject: [PATCH 16/18] added tests for queries in transactions --- tests/system/test_aggregation_query.py | 80 ++++++++++++++++++++++++-- tests/system/test_query.py | 31 ++++++++++ 2 files changed, 107 insertions(+), 4 deletions(-) diff --git a/tests/system/test_aggregation_query.py b/tests/system/test_aggregation_query.py index 05e7dadc..a473d4cc 100644 --- a/tests/system/test_aggregation_query.py +++ b/tests/system/test_aggregation_query.py @@ -84,6 +84,68 @@ def test_count_query_default(aggregation_query_client, nested_query, database_id assert r.value == expected_count +@pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) +@pytest.mark.parametrize( + "aggregation_type,aggregation_args,expected", + [ + ("count", (), len(populate_datastore.CHARACTERS)), + ( + "sum", + ("appearances",), + sum(c["appearances"] for c in populate_datastore.CHARACTERS), + ), + ( + "avg", + ("appearances",), + sum(c["appearances"] for c in populate_datastore.CHARACTERS) + / len(populate_datastore.CHARACTERS), + ), + ], +) +def test_aggregation_query_in_transaction( + aggregation_query_client, + nested_query, + database_id, + aggregation_type, + aggregation_args, + expected, +): + """ + When an aggregation query is run in a transaction, the transaction id should be sent with the request. + The result is the same as when it is run outside of a transaction. + """ + import mock + + with aggregation_query_client.transaction() as xact: + query = nested_query + + aggregation_query = aggregation_query_client.aggregation_query(query) + getattr(aggregation_query, aggregation_type)(*aggregation_args) + # Cannot use eventual consistency in a transaction + with pytest.raises(ValueError): + _do_fetch(aggregation_query, eventual=True) + # transaction id should be sent with query + with mock.patch.object( + aggregation_query_client._datastore_api, "run_aggregation_query" + ) as mock_api: + try: + _do_fetch(aggregation_query) + except ValueError: + # expect failure when parsing mock response + pass + assert mock_api.call_count == 1 + request = mock_api.call_args[1]["request"] + read_options = request["read_options"] + assert read_options.transaction == xact.id + # run full query + result = _do_fetch(aggregation_query) + assert len(result) == 1 + assert len(result[0]) == 1 + r = result[0][0] + assert r.alias == "property_1" + assert r.value == expected + + @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) def test_count_query_with_alias(aggregation_query_client, nested_query, database_id): query = nested_query @@ -190,13 +252,16 @@ def test_count_query_with_limit(aggregation_query_client, nested_query, database aggregation_query = aggregation_query_client.aggregation_query(query) aggregation_query.count(alias="total_high_limit") limit = 2 - result = _do_fetch(aggregation_query, limit=expected_count*2) # count with limit > total + result = _do_fetch( + aggregation_query, limit=expected_count * 2 + ) # count with limit > total assert len(result) == 1 assert len(result[0]) == 1 r = result[0][0] assert r.alias == "total_high_limit" assert r.value == expected_count + @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) def test_sum_query_with_limit(aggregation_query_client, nested_query, database_id): query = nested_query @@ -215,7 +280,9 @@ def test_sum_query_with_limit(aggregation_query_client, nested_query, database_i aggregation_query = aggregation_query_client.aggregation_query(query) aggregation_query.sum("appearances", alias="sum_high_limit") num_characters = len(populate_datastore.CHARACTERS) - result = _do_fetch(aggregation_query, limit=num_characters*2) # count with limit > total + result = _do_fetch( + aggregation_query, limit=num_characters * 2 + ) # count with limit > total assert len(result) == 1 assert len(result[0]) == 1 r = result[0][0] @@ -243,12 +310,17 @@ def test_avg_query_with_limit(aggregation_query_client, nested_query, database_i aggregation_query = aggregation_query_client.aggregation_query(query) aggregation_query.avg("appearances", alias="avg_high_limit") num_characters = len(populate_datastore.CHARACTERS) - result = _do_fetch(aggregation_query, limit=num_characters*2) # count with limit > total + result = _do_fetch( + aggregation_query, limit=num_characters * 2 + ) # count with limit > total assert len(result) == 1 assert len(result[0]) == 1 r = result[0][0] assert r.alias == "avg_high_limit" - assert r.value == sum(c["appearances"] for c in populate_datastore.CHARACTERS) / num_characters + assert ( + r.value + == sum(c["appearances"] for c in populate_datastore.CHARACTERS) / num_characters + ) @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) diff --git a/tests/system/test_query.py b/tests/system/test_query.py index 864bab57..286cf929 100644 --- a/tests/system/test_query.py +++ b/tests/system/test_query.py @@ -82,6 +82,37 @@ def test_query_w_ancestor(ancestor_query, database_id): assert len(entities) == expected_matches +@pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) +def test_query_in_transaction(ancestor_query, database_id): + """ + when a query is run in a transaction, the transaction id should be sent with the request. + the result is the same as when it is run outside of a transaction. + """ + import mock + + query = ancestor_query + client = query._client + expected_matches = 8 + with client.transaction() as xact: + # cannot use eventual consistency in a transaction + with pytest.raises(ValueError): + _do_fetch(query, eventual=True) + # transaction id should be sent with query + with mock.patch.object(client._datastore_api, "run_query") as mock_api: + try: + _do_fetch(query) + except TypeError: + # expect failure when parsing mock response + pass + assert mock_api.call_count == 1 + request = mock_api.call_args[1]["request"] + read_options = request["read_options"] + assert read_options.transaction == xact.id + # run full query + entities = _do_fetch(query) + assert len(entities) == expected_matches + + @pytest.mark.parametrize("database_id", [None, _helpers.TEST_DATABASE], indirect=True) def test_query_w_limit_paging(ancestor_query, database_id): query = ancestor_query From 61a3f240ac5d36765e7954e1c2a8e18b111ddd8d Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 31 Aug 2023 12:36:55 -0700 Subject: [PATCH 17/18] broke system test into unit tests --- tests/system/test_aggregation_query.py | 20 +------ tests/system/test_query.py | 18 +----- tests/unit/test_aggregation.py | 79 +++++++++++++++++++++++++- tests/unit/test_query.py | 50 ++++++++++++++++ 4 files changed, 130 insertions(+), 37 deletions(-) diff --git a/tests/system/test_aggregation_query.py b/tests/system/test_aggregation_query.py index a473d4cc..ae9a8297 100644 --- a/tests/system/test_aggregation_query.py +++ b/tests/system/test_aggregation_query.py @@ -114,29 +114,11 @@ def test_aggregation_query_in_transaction( When an aggregation query is run in a transaction, the transaction id should be sent with the request. The result is the same as when it is run outside of a transaction. """ - import mock - - with aggregation_query_client.transaction() as xact: + with aggregation_query_client.transaction(): query = nested_query aggregation_query = aggregation_query_client.aggregation_query(query) getattr(aggregation_query, aggregation_type)(*aggregation_args) - # Cannot use eventual consistency in a transaction - with pytest.raises(ValueError): - _do_fetch(aggregation_query, eventual=True) - # transaction id should be sent with query - with mock.patch.object( - aggregation_query_client._datastore_api, "run_aggregation_query" - ) as mock_api: - try: - _do_fetch(aggregation_query) - except ValueError: - # expect failure when parsing mock response - pass - assert mock_api.call_count == 1 - request = mock_api.call_args[1]["request"] - read_options = request["read_options"] - assert read_options.transaction == xact.id # run full query result = _do_fetch(aggregation_query) assert len(result) == 1 diff --git a/tests/system/test_query.py b/tests/system/test_query.py index 286cf929..9d7bec06 100644 --- a/tests/system/test_query.py +++ b/tests/system/test_query.py @@ -88,26 +88,10 @@ def test_query_in_transaction(ancestor_query, database_id): when a query is run in a transaction, the transaction id should be sent with the request. the result is the same as when it is run outside of a transaction. """ - import mock - query = ancestor_query client = query._client expected_matches = 8 - with client.transaction() as xact: - # cannot use eventual consistency in a transaction - with pytest.raises(ValueError): - _do_fetch(query, eventual=True) - # transaction id should be sent with query - with mock.patch.object(client._datastore_api, "run_query") as mock_api: - try: - _do_fetch(query) - except TypeError: - # expect failure when parsing mock response - pass - assert mock_api.call_count == 1 - request = mock_api.call_args[1]["request"] - read_options = request["read_options"] - assert read_options.transaction == xact.id + with client.transaction(): # run full query entities = _do_fetch(query) assert len(entities) == expected_matches diff --git a/tests/unit/test_aggregation.py b/tests/unit/test_aggregation.py index 123433bc..15d11aca 100644 --- a/tests/unit/test_aggregation.py +++ b/tests/unit/test_aggregation.py @@ -537,6 +537,81 @@ def test__item_to_aggregation_result(): assert result[0].value == map_composite_mock.__getitem__().integer_value +@pytest.mark.parametrize("database_id", [None, "somedb"], indirect=True) +@pytest.mark.parametrize( + "aggregation_type,aggregation_args", + [ + ("count", ()), + ( + "sum", + ("appearances",), + ), + ("avg", ("appearances",)), + ], +) +def test_eventual_transaction_fails(database_id, aggregation_type, aggregation_args): + """ + Queries with eventual consistency cannot be used in a transaction. + """ + import mock + + transaction = mock.Mock() + transaction.id = b"expected_id" + client = _Client(None, database=database_id, transaction=transaction) + + query = _make_query(client) + aggregation_query = _make_aggregation_query(client=client, query=query) + # initiate requested aggregation (ex count, sum, avg) + getattr(aggregation_query, aggregation_type)(*aggregation_args) + with pytest.raises(ValueError): + list(aggregation_query.fetch(eventual=True)) + + +@pytest.mark.parametrize("database_id", [None, "somedb"], indirect=True) +@pytest.mark.parametrize( + "aggregation_type,aggregation_args", + [ + ("count", ()), + ( + "sum", + ("appearances",), + ), + ("avg", ("appearances",)), + ], +) +def test_transaction_id_populated(database_id, aggregation_type, aggregation_args): + """ + When an aggregation is run in the context of a transaction, the transaction + ID should be populated in the request. + """ + import mock + + transaction = mock.Mock() + transaction.id = b"expected_id" + mock_datastore_api = mock.Mock() + mock_gapic = mock_datastore_api.run_aggregation_query + mock_gapic.return_value = _make_aggregation_query_response([]) + client = _Client( + None, + datastore_api=mock_datastore_api, + database=database_id, + transaction=transaction, + ) + + query = _make_query(client) + aggregation_query = _make_aggregation_query(client=client, query=query) + + # initiate requested aggregation (ex count, sum, avg) + getattr(aggregation_query, aggregation_type)(*aggregation_args) + # run mock query + list(aggregation_query.fetch()) + assert mock_gapic.call_count == 1 + request = mock_gapic.call_args[1]["request"] + read_options = request["read_options"] + # ensure transaction ID is populated + assert read_options.transaction == client.current_transaction.id + + class _Client(object): def __init__( self, @@ -570,7 +645,9 @@ def _make_aggregation_iterator(*args, **kw): return AggregationResultIterator(*args, **kw) -def _make_aggregation_query_response(aggregation_pbs, more_results_enum): +def _make_aggregation_query_response( + aggregation_pbs, more_results_enum=3 +): # 3 = NO_MORE_RESULTS from google.cloud.datastore_v1.types import datastore as datastore_pb2 from google.cloud.datastore_v1.types import aggregation_result diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py index 25b3febb..7758d7fb 100644 --- a/tests/unit/test_query.py +++ b/tests/unit/test_query.py @@ -652,6 +652,56 @@ def test_query_fetch_w_explicit_client_w_retry_w_timeout(database_id): assert iterator._timeout == timeout +@pytest.mark.parametrize("database_id", [None, "somedb"]) +def test_eventual_transaction_fails(database_id): + """ + Queries with eventual consistency cannot be used in a transaction. + """ + import mock + + transaction = mock.Mock() + transaction.id = b"expected_id" + client = _Client(None, database=database_id, transaction=transaction) + + query = _make_query(client) + with pytest.raises(ValueError): + list(query.fetch(eventual=True)) + + +@pytest.mark.parametrize("database_id", [None, "somedb"]) +def test_transaction_id_populated(database_id): + """ + When an aggregation is run in the context of a transaction, the transaction + ID should be populated in the request. + """ + import mock + + transaction = mock.Mock() + transaction.id = b"expected_id" + mock_datastore_api = mock.Mock() + mock_gapic = mock_datastore_api.run_query + + more_results_enum = 3 # NO_MORE_RESULTS + response_pb = _make_query_response([], b"", more_results_enum, 0) + mock_gapic.return_value = response_pb + + client = _Client( + None, + datastore_api=mock_datastore_api, + database=database_id, + transaction=transaction, + ) + + query = _make_query(client) + # run mock query + list(query.fetch()) + assert mock_gapic.call_count == 1 + request = mock_gapic.call_args[1]["request"] + read_options = request["read_options"] + # ensure transaction ID is populated + assert read_options.transaction == client.current_transaction.id + + def test_iterator_constructor_defaults(): query = object() client = object() From 327820f51756722d55002e2fb7df83d49657006a Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 31 Aug 2023 14:28:24 -0700 Subject: [PATCH 18/18] removed duplicate line --- google/cloud/datastore/aggregation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/google/cloud/datastore/aggregation.py b/google/cloud/datastore/aggregation.py index e2198270..0518514e 100644 --- a/google/cloud/datastore/aggregation.py +++ b/google/cloud/datastore/aggregation.py @@ -95,7 +95,6 @@ def _to_pb(self): Convert this instance to the protobuf representation """ aggregation_pb = query_pb2.AggregationQuery.Aggregation() - aggregation_pb.alias = self.alias aggregation_pb.sum = query_pb2.AggregationQuery.Aggregation.Sum() aggregation_pb.sum.property.name = self.property_ref aggregation_pb.alias = self.alias