From 0b5c1d597cdec3a05a16fb935595f773c5840bd4 Mon Sep 17 00:00:00 2001 From: shollyman Date: Tue, 16 Jan 2024 11:09:40 -0800 Subject: [PATCH 01/11] feat: support universe resolution (#1774) * feat: support universe resolution This PR wires up consumption of the universe_domain client option for resolving the endpoint for constructing the BQ client. Testing universes is not yet something we want to in this repo, so validation was done out of band. * formatting and testing * conditionals for stale core * formatting * unused import --- google/cloud/bigquery/_helpers.py | 3 +++ google/cloud/bigquery/client.py | 9 +++++++++ tests/unit/test_client.py | 17 +++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/google/cloud/bigquery/_helpers.py b/google/cloud/bigquery/_helpers.py index 4cf6dddac..905d4aee1 100644 --- a/google/cloud/bigquery/_helpers.py +++ b/google/cloud/bigquery/_helpers.py @@ -55,6 +55,9 @@ _DEFAULT_HOST = "https://bigquery.googleapis.com" """Default host for JSON API.""" +_DEFAULT_UNIVERSE = "googleapis.com" +"""Default universe for the JSON API.""" + def _get_bigquery_host(): return os.environ.get(BIGQUERY_EMULATOR_HOST, _DEFAULT_HOST) diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index 182319646..b2ea130c4 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -78,6 +78,7 @@ from google.cloud.bigquery._helpers import _verify_job_config_type from google.cloud.bigquery._helpers import _get_bigquery_host from google.cloud.bigquery._helpers import _DEFAULT_HOST +from google.cloud.bigquery._helpers import _DEFAULT_UNIVERSE from google.cloud.bigquery._job_helpers import make_job_id as _make_job_id from google.cloud.bigquery.dataset import Dataset from google.cloud.bigquery.dataset import DatasetListItem @@ -252,6 +253,14 @@ def __init__( if client_options.api_endpoint: api_endpoint = client_options.api_endpoint kw_args["api_endpoint"] = api_endpoint + elif ( + hasattr(client_options, "universe_domain") + and client_options.universe_domain + and client_options.universe_domain is not _DEFAULT_UNIVERSE + ): + kw_args["api_endpoint"] = _DEFAULT_HOST.replace( + _DEFAULT_UNIVERSE, client_options.universe_domain + ) self._connection = Connection(self, **kw_args) self._location = location diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index ad22e0ddb..56bdbad5e 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -201,6 +201,23 @@ def test_ctor_w_client_options_object(self): client._connection.API_BASE_URL, "https://www.foo-googleapis.com" ) + @pytest.mark.skipif( + packaging.version.parse(getattr(google.api_core, "__version__", "0.0.0")) + < packaging.version.Version("2.15.0"), + reason="universe_domain not supported with google-api-core < 2.15.0", + ) + def test_ctor_w_client_options_universe(self): + creds = _make_credentials() + http = object() + client_options = {"universe_domain": "foo.com"} + client = self._make_one( + project=self.PROJECT, + credentials=creds, + _http=http, + client_options=client_options, + ) + self.assertEqual(client._connection.API_BASE_URL, "https://bigquery.foo.com") + def test_ctor_w_location(self): from google.cloud.bigquery._http import Connection From 1271b18f14efb2d8c9bb3a5b35db2e949111cec5 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 16 Jan 2024 22:26:15 +0000 Subject: [PATCH 02/11] build(python): fix `docs` and `docfx` builds (#1779) Source-Link: https://togithub.com/googleapis/synthtool/commit/fac8444edd5f5526e804c306b766a271772a3e2f Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:5ea6d0ab82c956b50962f91d94e206d3921537ae5fe1549ec5326381d8905cfa --- .github/.OwlBot.lock.yaml | 6 +- .kokoro/requirements.txt | 6 +- docs/reference.rst | 115 ++++---------------------------------- noxfile.py | 24 +++++++- 4 files changed, 39 insertions(+), 112 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 773c1dfd2..d8a1bbca7 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2024 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:2f155882785883336b4468d5218db737bb1d10c9cea7cb62219ad16fe248c03c -# created: 2023-11-29T14:54:29.548172703Z + digest: sha256:5ea6d0ab82c956b50962f91d94e206d3921537ae5fe1549ec5326381d8905cfa +# created: 2024-01-15T16:32:08.142785673Z diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index e5c1ffca9..bb3d6ca38 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -263,9 +263,9 @@ jeepney==0.8.0 \ # via # keyring # secretstorage -jinja2==3.1.2 \ - --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \ - --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61 +jinja2==3.1.3 \ + --hash=sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa \ + --hash=sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90 # via gcp-releasetool keyring==24.2.0 \ --hash=sha256:4901caaf597bfd3bbd78c9a0c7c4c29fcd8310dab2cffefe749e916b6527acd6 \ diff --git a/docs/reference.rst b/docs/reference.rst index b886f1161..6c00df077 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -1,8 +1,6 @@ API Reference ~~~~~~~~~~~~~ -.. currentmodule:: google.cloud.bigquery - The main concepts with this API are: - :class:`~google.cloud.bigquery.client.Client` manages connections to the @@ -18,55 +16,12 @@ The main concepts with this API are: Client ====== -.. autosummary:: - :toctree: generated - - client.Client +.. automodule:: google.cloud.bigquery.client Job === -Job Configuration ------------------ - -.. autosummary:: - :toctree: generated - - job.QueryJobConfig - job.CopyJobConfig - job.LoadJobConfig - job.ExtractJobConfig - -Job Classes ------------ - -.. autosummary:: - :toctree: generated - - job.QueryJob - job.CopyJob - job.LoadJob - job.ExtractJob - -Job-Related Types ------------------ - -.. autosummary:: - :toctree: generated - - job.Compression - job.CreateDisposition - job.DestinationFormat - job.DmlStats - job.Encoding - job.OperationType - job.QueryPlanEntry - job.QueryPlanEntryStep - job.QueryPriority - job.ReservationUsage - job.SourceFormat - job.WriteDisposition - job.SchemaUpdateOption +.. automodule:: google.cloud.bigquery.job .. toctree:: :maxdepth: 2 @@ -77,63 +32,28 @@ Job-Related Types Dataset ======= -.. autosummary:: - :toctree: generated - - dataset.Dataset - dataset.DatasetListItem - dataset.DatasetReference - dataset.AccessEntry +.. automodule:: google.cloud.bigquery.dataset Table ===== -.. autosummary:: - :toctree: generated - - table.PartitionRange - table.RangePartitioning - table.Row - table.RowIterator - table.SnapshotDefinition - table.CloneDefinition - table.Table - table.TableListItem - table.TableReference - table.TimePartitioning - table.TimePartitioningType +.. automodule:: google.cloud.bigquery.table Model ===== -.. autosummary:: - :toctree: generated - - model.Model - model.ModelReference +.. automodule:: google.cloud.bigquery.model Routine ======= -.. autosummary:: - :toctree: generated - - routine.DeterminismLevel - routine.Routine - routine.RoutineArgument - routine.RoutineReference - routine.RoutineType +.. automodule:: google.cloud.bigquery.routine Schema ====== -.. autosummary:: - :toctree: generated - - schema.SchemaField - schema.PolicyTagList - +.. automodule:: google.cloud.bigquery.schema Query ===== @@ -147,25 +67,13 @@ Query Retries ======= -.. autosummary:: - :toctree: generated - - retry.DEFAULT_RETRY +.. automodule:: google.cloud.bigquery.retry External Configuration ====================== -.. autosummary:: - :toctree: generated - - external_config.ExternalSourceFormat - external_config.ExternalConfig - external_config.BigtableOptions - external_config.BigtableColumnFamily - external_config.BigtableColumn - external_config.CSVOptions - external_config.GoogleSheetsOptions +.. automodule:: google.cloud.bigquery.external_config .. toctree:: :maxdepth: 2 @@ -194,10 +102,7 @@ Enums Encryption Configuration ======================== -.. autosummary:: - :toctree: generated - - encryption_configuration.EncryptionConfiguration +.. automodule:: google.cloud.bigquery.encryption_configuration Additional Types diff --git a/noxfile.py b/noxfile.py index 66d68c04e..ae022232e 100644 --- a/noxfile.py +++ b/noxfile.py @@ -418,7 +418,20 @@ def blacken(session): def docs(session): """Build the docs.""" - session.install("recommonmark", "sphinx==4.0.2", "sphinx_rtd_theme") + session.install( + # We need to pin to specific versions of the `sphinxcontrib-*` packages + # which still support sphinx 4.x. + # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 + # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. + "sphinxcontrib-applehelp==1.0.4", + "sphinxcontrib-devhelp==1.0.2", + "sphinxcontrib-htmlhelp==2.0.1", + "sphinxcontrib-qthelp==1.0.3", + "sphinxcontrib-serializinghtml==1.1.5", + "sphinx==4.5.0", + "alabaster", + "recommonmark", + ) session.install("google-cloud-storage") session.install("-e", ".[all]") @@ -443,6 +456,15 @@ def docfx(session): session.install("-e", ".") session.install( + # We need to pin to specific versions of the `sphinxcontrib-*` packages + # which still support sphinx 4.x. + # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 + # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. + "sphinxcontrib-applehelp==1.0.4", + "sphinxcontrib-devhelp==1.0.2", + "sphinxcontrib-htmlhelp==2.0.1", + "sphinxcontrib-qthelp==1.0.3", + "sphinxcontrib-serializinghtml==1.1.5", "gcp-sphinx-docfx-yaml", "alabaster", "recommonmark", From d90602de87e58b665cb974401a327a640805822f Mon Sep 17 00:00:00 2001 From: Stephanie A <129541811+DevStephanie@users.noreply.github.com> Date: Tue, 16 Jan 2024 17:04:16 -0600 Subject: [PATCH 03/11] docs: update `snippets.py` to use `query_and_wait` (#1773) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://togithub.com/googleapis/python-bigquery/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) Fixes # 🦕 --- docs/snippets.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/snippets.py b/docs/snippets.py index 72ac2a000..b4e78e36f 100644 --- a/docs/snippets.py +++ b/docs/snippets.py @@ -465,13 +465,12 @@ def test_client_query_total_rows(client, capsys): 'WHERE state = "TX" ' "LIMIT 100" ) - query_job = client.query( + results = client.query_and_wait( query, # Location must match that of the dataset(s) referenced in the query. location="US", - ) # API request - starts the query + ) # API request - starts the query and waits for results. - results = query_job.result() # Wait for query to complete. print("Got {} rows.".format(results.total_rows)) # [END bigquery_query_total_rows] @@ -551,7 +550,7 @@ def test_query_results_as_dataframe(client): LIMIT 10 """ - df = client.query(sql).to_dataframe() + df = client.query_and_wait(sql).to_dataframe() # [END bigquery_query_results_dataframe] assert isinstance(df, pandas.DataFrame) assert len(list(df)) == 2 # verify the number of columns From ffe80599429bef17681a37ec34e03488449ea812 Mon Sep 17 00:00:00 2001 From: Kira Date: Wed, 17 Jan 2024 15:08:00 -0500 Subject: [PATCH 04/11] chore: cleanup resources at startup (#1741) * chore: cleanup resources at startup time * reformmated with black for linter * changd how to call prefixer to clean up datasets, not tables * Removed formatting for uuid * Removed unneeded import of uuid * remove comment from dataset_access_test.py --- samples/snippets/authorized_view_tutorial_test.py | 10 +++------- samples/snippets/materialized_view_test.py | 3 ++- samples/snippets/natality_tutorial_test.py | 8 ++------ samples/snippets/quickstart_test.py | 6 ++---- samples/snippets/view_test.py | 3 ++- 5 files changed, 11 insertions(+), 19 deletions(-) diff --git a/samples/snippets/authorized_view_tutorial_test.py b/samples/snippets/authorized_view_tutorial_test.py index e2220fb54..04f6312d3 100644 --- a/samples/snippets/authorized_view_tutorial_test.py +++ b/samples/snippets/authorized_view_tutorial_test.py @@ -13,12 +13,12 @@ # limitations under the License. from typing import Iterator, List -import uuid from google.cloud import bigquery import pytest import authorized_view_tutorial # type: ignore +from conftest import prefixer # type: ignore @pytest.fixture(scope="module") @@ -38,12 +38,8 @@ def test_authorized_view_tutorial( client: bigquery.Client, datasets_to_delete: List[str] ) -> None: override_values = { - "source_dataset_id": "github_source_data_{}".format( - str(uuid.uuid4()).replace("-", "_") - ), - "shared_dataset_id": "shared_views_{}".format( - str(uuid.uuid4()).replace("-", "_") - ), + "source_dataset_id": f"{prefixer.create_prefix()}_authorized_view_tutorial", + "shared_dataset_id": f"{prefixer.create_prefix()}_authorized_view_tutorial_shared_views", } source_dataset_ref = "{}.{}".format( client.project, override_values["source_dataset_id"] diff --git a/samples/snippets/materialized_view_test.py b/samples/snippets/materialized_view_test.py index 59e08131e..1b464af6f 100644 --- a/samples/snippets/materialized_view_test.py +++ b/samples/snippets/materialized_view_test.py @@ -21,6 +21,7 @@ import pytest import materialized_view # type: ignore +from conftest import prefixer # type: ignore def temp_suffix() -> str: @@ -37,7 +38,7 @@ def bigquery_client_patch( @pytest.fixture(scope="module") def dataset_id(bigquery_client: bigquery.Client) -> Iterator[str]: - dataset_id = f"mvdataset_{temp_suffix()}" + dataset_id = f"{prefixer.create_prefix()}_materialized_view" bigquery_client.create_dataset(dataset_id) yield dataset_id bigquery_client.delete_dataset(dataset_id, delete_contents=True) diff --git a/samples/snippets/natality_tutorial_test.py b/samples/snippets/natality_tutorial_test.py index 7f24ca5cb..603d142f2 100644 --- a/samples/snippets/natality_tutorial_test.py +++ b/samples/snippets/natality_tutorial_test.py @@ -13,12 +13,12 @@ # limitations under the License. from typing import Iterator, List -import uuid from google.cloud import bigquery import pytest import natality_tutorial # type: ignore +from conftest import prefixer # type: ignore @pytest.fixture(scope="module") @@ -37,11 +37,7 @@ def datasets_to_delete(client: bigquery.Client) -> Iterator[List[str]]: def test_natality_tutorial( client: bigquery.Client, datasets_to_delete: List[str] ) -> None: - override_values = { - "dataset_id": "natality_regression_{}".format( - str(uuid.uuid4()).replace("-", "_") - ), - } + override_values = {"dataset_id": f"{prefixer.create_prefix()}_natality_tutorial"} datasets_to_delete.append(override_values["dataset_id"]) natality_tutorial.run_natality_tutorial(override_values) diff --git a/samples/snippets/quickstart_test.py b/samples/snippets/quickstart_test.py index 88a24618d..74a02a83a 100644 --- a/samples/snippets/quickstart_test.py +++ b/samples/snippets/quickstart_test.py @@ -13,12 +13,12 @@ # limitations under the License. from typing import Iterator, List -import uuid from google.cloud import bigquery import pytest import quickstart # type: ignore +from conftest import prefixer # type: ignore # Must match the dataset listed in quickstart.py (there's no easy way to # extract this). @@ -43,9 +43,7 @@ def test_quickstart( client: bigquery.Client, datasets_to_delete: List[str], ) -> None: - override_values = { - "dataset_id": "my_new_dataset_{}".format(str(uuid.uuid4()).replace("-", "_")), - } + override_values = {"dataset_id": f"{prefixer.create_prefix()}_quickstart"} datasets_to_delete.append(override_values["dataset_id"]) quickstart.run_quickstart(override_values) diff --git a/samples/snippets/view_test.py b/samples/snippets/view_test.py index 1e615db47..dfa1cdeee 100644 --- a/samples/snippets/view_test.py +++ b/samples/snippets/view_test.py @@ -20,6 +20,7 @@ import pytest import view # type: ignore +from conftest import prefixer # type: ignore def temp_suffix() -> str: @@ -53,7 +54,7 @@ def view_id(bigquery_client: bigquery.Client, view_dataset_id: str) -> Iterator[ def source_dataset_id( bigquery_client: bigquery.Client, project_id: str ) -> Iterator[str]: - dataset_id = f"{project_id}.view_{temp_suffix()}" + dataset_id = f"{prefixer.create_prefix()}_view" bigquery_client.create_dataset(dataset_id) yield dataset_id bigquery_client.delete_dataset(dataset_id, delete_contents=True) From d1161dddde41a7d35b30033ccbf6984a5de640bd Mon Sep 17 00:00:00 2001 From: Kira Date: Wed, 17 Jan 2024 15:53:20 -0500 Subject: [PATCH 05/11] docs: update multiple samples to change query to query_and_wait (#1784) * docs: update multiple samples for query_and_wait API * black * update rest of samples to use query_and_wait * changed query_jobs to results --- samples/client_query_add_column.py | 5 ++--- samples/client_query_destination_table_clustered.py | 5 +++-- samples/client_query_legacy_sql.py | 8 +++++--- samples/client_query_relax_column.py | 5 ++--- samples/client_query_w_struct_params.py | 6 ++++-- samples/download_public_data_sandbox.py | 4 +++- samples/snippets/authorized_view_tutorial.py | 6 ++---- samples/snippets/natality_tutorial.py | 3 +-- samples/snippets/simple_app.py | 6 ++---- samples/tests/conftest.py | 2 +- 10 files changed, 25 insertions(+), 25 deletions(-) diff --git a/samples/client_query_add_column.py b/samples/client_query_add_column.py index ec14087fb..6aae5fce4 100644 --- a/samples/client_query_add_column.py +++ b/samples/client_query_add_column.py @@ -36,14 +36,13 @@ def client_query_add_column(table_id: str) -> None: ) # Start the query, passing in the extra configuration. - query_job = client.query( + client.query_and_wait( # In this example, the existing table contains only the 'full_name' and # 'age' columns, while the results of this query will contain an # additional 'favorite_color' column. 'SELECT "Timmy" as full_name, 85 as age, "Blue" as favorite_color;', job_config=job_config, - ) # Make an API request. - query_job.result() # Wait for the job to complete. + ) # Make an API request and wait for job to complete. # Checks the updated length of the schema. table = client.get_table(table_id) # Make an API request. diff --git a/samples/client_query_destination_table_clustered.py b/samples/client_query_destination_table_clustered.py index de9fff2d0..19330500a 100644 --- a/samples/client_query_destination_table_clustered.py +++ b/samples/client_query_destination_table_clustered.py @@ -31,8 +31,9 @@ def client_query_destination_table_clustered(table_id: str) -> None: ) # Start the query, passing in the extra configuration. - query_job = client.query(sql, job_config=job_config) # Make an API request. - query_job.result() # Wait for the job to complete. + client.query_and_wait( + sql, job_config=job_config + ) # Make an API request and wait for job to complete. table = client.get_table(table_id) # Make an API request. if table.clustering_fields == cluster_fields: diff --git a/samples/client_query_legacy_sql.py b/samples/client_query_legacy_sql.py index 44917e4e0..1fb5b797a 100644 --- a/samples/client_query_legacy_sql.py +++ b/samples/client_query_legacy_sql.py @@ -29,10 +29,12 @@ def client_query_legacy_sql() -> None: # Set use_legacy_sql to True to use legacy SQL syntax. job_config = bigquery.QueryJobConfig(use_legacy_sql=True) - # Start the query, passing in the extra configuration. - query_job = client.query(query, job_config=job_config) # Make an API request. + # Start the query and waits for query job to complete, passing in the extra configuration. + results = client.query_and_wait( + query, job_config=job_config + ) # Make an API request. print("The query data:") - for row in query_job: + for row in results: print(row) # [END bigquery_query_legacy] diff --git a/samples/client_query_relax_column.py b/samples/client_query_relax_column.py index 22ecb33d1..26dce888f 100644 --- a/samples/client_query_relax_column.py +++ b/samples/client_query_relax_column.py @@ -39,13 +39,12 @@ def client_query_relax_column(table_id: str) -> None: ) # Start the query, passing in the extra configuration. - query_job = client.query( + client.query_and_wait( # In this example, the existing table contains 'full_name' and 'age' as # required columns, but the query results will omit the second column. 'SELECT "Beyonce" as full_name;', job_config=job_config, - ) # Make an API request. - query_job.result() # Wait for the job to complete. + ) # Make an API request and wait for job to complete # Checks the updated number of required fields. table = client.get_table(table_id) # Make an API request. diff --git a/samples/client_query_w_struct_params.py b/samples/client_query_w_struct_params.py index 6b68e78ed..cda2fcb43 100644 --- a/samples/client_query_w_struct_params.py +++ b/samples/client_query_w_struct_params.py @@ -30,8 +30,10 @@ def client_query_w_struct_params() -> None: ) ] ) - query_job = client.query(query, job_config=job_config) # Make an API request. + results = client.query_and_wait( + query, job_config=job_config + ) # Make an API request and waits for results. - for row in query_job: + for row in results: print(row.s) # [END bigquery_query_params_structs] diff --git a/samples/download_public_data_sandbox.py b/samples/download_public_data_sandbox.py index e165a31ce..909a7da05 100644 --- a/samples/download_public_data_sandbox.py +++ b/samples/download_public_data_sandbox.py @@ -27,7 +27,9 @@ def download_public_data_sandbox() -> None: query_string = "SELECT * FROM `bigquery-public-data.usa_names.usa_1910_current`" # Use the BigQuery Storage API to speed-up downloads of large tables. - dataframe = client.query(query_string).to_dataframe(create_bqstorage_client=True) + dataframe = client.query_and_wait(query_string).to_dataframe( + create_bqstorage_client=True + ) print(dataframe.info()) # [END bigquery_pandas_public_data_sandbox] diff --git a/samples/snippets/authorized_view_tutorial.py b/samples/snippets/authorized_view_tutorial.py index bfb61bc38..f52170bc6 100644 --- a/samples/snippets/authorized_view_tutorial.py +++ b/samples/snippets/authorized_view_tutorial.py @@ -62,15 +62,13 @@ def run_authorized_view_tutorial( FROM `bigquery-public-data.github_repos.commits` LIMIT 1000 """ - query_job = client.query( + client.query_and_wait( sql, # Location must match that of the dataset(s) referenced in the query # and of the destination table. location="US", job_config=job_config, - ) # API request - starts the query - - query_job.result() # Waits for the query to finish + ) # API request - starts the query and waits for query to finish # [END bigquery_avt_create_source_table] # Create a separate dataset to store your view diff --git a/samples/snippets/natality_tutorial.py b/samples/snippets/natality_tutorial.py index b330a3c21..df9fc15be 100644 --- a/samples/snippets/natality_tutorial.py +++ b/samples/snippets/natality_tutorial.py @@ -83,8 +83,7 @@ def run_natality_tutorial(override_values: Optional[Dict[str, str]] = None) -> N """ # Run the query. - query_job = client.query(query, job_config=job_config) - query_job.result() # Waits for the query to finish + client.query_and_wait(query, job_config=job_config) # Waits for the query to finish # [END bigquery_query_natality_tutorial] diff --git a/samples/snippets/simple_app.py b/samples/snippets/simple_app.py index 3d856d4bb..8281e1877 100644 --- a/samples/snippets/simple_app.py +++ b/samples/snippets/simple_app.py @@ -27,7 +27,7 @@ def query_stackoverflow() -> None: client = bigquery.Client() # [END bigquery_simple_app_client] # [START bigquery_simple_app_query] - query_job = client.query( + results = client.query_and_wait( """ SELECT CONCAT( @@ -38,9 +38,7 @@ def query_stackoverflow() -> None: WHERE tags like '%google-bigquery%' ORDER BY view_count DESC LIMIT 10""" - ) - - results = query_job.result() # Waits for job to complete. + ) # Waits for job to complete. # [END bigquery_simple_app_query] # [START bigquery_simple_app_print] diff --git a/samples/tests/conftest.py b/samples/tests/conftest.py index 99bd2e367..2b5b89c43 100644 --- a/samples/tests/conftest.py +++ b/samples/tests/conftest.py @@ -174,7 +174,7 @@ def model_id(client: bigquery.Client, dataset_id: str) -> str: model_id ) - client.query(sql).result() + client.query_and_wait(sql) return model_id From 955a4cd99e21cbca1b2f9c1dc6aa3fd8070cd61f Mon Sep 17 00:00:00 2001 From: Salem Jorden <115185670+SalemJorden@users.noreply.github.com> Date: Thu, 18 Jan 2024 12:28:28 -0600 Subject: [PATCH 06/11] docs: update the query with no cache sample to use query_and_wait API (#1770) Co-authored-by: Salem Boyland Co-authored-by: Kira --- samples/query_no_cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/query_no_cache.py b/samples/query_no_cache.py index 7501b7fc0..b942e5010 100644 --- a/samples/query_no_cache.py +++ b/samples/query_no_cache.py @@ -26,8 +26,8 @@ def query_no_cache() -> None: FROM `bigquery-public-data.samples.shakespeare` GROUP BY corpus; """ - query_job = client.query(sql, job_config=job_config) # Make an API request. + results = client.query_and_wait(sql, job_config=job_config) # Make an API request. - for row in query_job: + for row in results: print(row) # [END bigquery_query_no_cache] From 1f96439b3dbd27f11be5e2af84f290ec6094d0a4 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 19 Jan 2024 17:35:26 -0600 Subject: [PATCH 07/11] docs: remove unused query code sample (#1769) * docs: remove unused query code sample This sample was moved in https://github.com/googleapis/python-bigquery/pull/1722/files#diff-2e8df14049580f42d6c73a3209838b96f3c9b185d2d7f2688683ae60bb2e7c43. Docs updated in internal change 597332356. * remove sample test too * update reference to query() sample in usage guides --------- Co-authored-by: Kira --- docs/usage/queries.rst | 4 +-- samples/client_query.py | 41 ------------------------------ samples/tests/test_client_query.py | 27 -------------------- 3 files changed, 2 insertions(+), 70 deletions(-) delete mode 100644 samples/client_query.py delete mode 100644 samples/tests/test_client_query.py diff --git a/docs/usage/queries.rst b/docs/usage/queries.rst index fc57e54de..56be8497e 100644 --- a/docs/usage/queries.rst +++ b/docs/usage/queries.rst @@ -5,9 +5,9 @@ Querying data ^^^^^^^^^^^^^ Run a query and wait for it to finish with the -:func:`~google.cloud.bigquery.client.Client.query` method: +:func:`~google.cloud.bigquery.client.Client.query_and_wait` method: -.. literalinclude:: ../samples/client_query.py +.. literalinclude:: ../samples/snippets/client_query.py :language: python :dedent: 4 :start-after: [START bigquery_query] diff --git a/samples/client_query.py b/samples/client_query.py deleted file mode 100644 index 80eac854e..000000000 --- a/samples/client_query.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -def client_query() -> None: - # TODO(swast): remove once docs in cloud.google.com have been updated to - # use samples/snippets/client_query.py - - # [START bigquery_query] - - from google.cloud import bigquery - - # Construct a BigQuery client object. - client = bigquery.Client() - - query = """ - SELECT name, SUM(number) as total_people - FROM `bigquery-public-data.usa_names.usa_1910_2013` - WHERE state = 'TX' - GROUP BY name, state - ORDER BY total_people DESC - LIMIT 20 - """ - query_job = client.query(query) # Make an API request. - - print("The query data:") - for row in query_job: - # Row values can be accessed by field name or index. - print("name={}, count={}".format(row[0], row["total_people"])) - # [END bigquery_query] diff --git a/samples/tests/test_client_query.py b/samples/tests/test_client_query.py deleted file mode 100644 index 5d4fb9c94..000000000 --- a/samples/tests/test_client_query.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import typing - -from .. import client_query - -if typing.TYPE_CHECKING: - import pytest - - -def test_client_query(capsys: "pytest.CaptureFixture[str]") -> None: - client_query.client_query() - out, err = capsys.readouterr() - assert "The query data:" in out - assert "name=James, count=272793" in out From 89f1299b3164b51fb0f29bc600a34ded59c10682 Mon Sep 17 00:00:00 2001 From: Stephanie A <129541811+DevStephanie@users.noreply.github.com> Date: Mon, 22 Jan 2024 15:18:16 -0600 Subject: [PATCH 08/11] docs: Updates `query` to `query and wait` in samples/desktopapp/user_credentials.py (#1787) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Updates file * Updates files * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Updates * edits --------- Co-authored-by: Owl Bot --- samples/desktopapp/user_credentials.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/desktopapp/user_credentials.py b/samples/desktopapp/user_credentials.py index 487a56c5f..68236d126 100644 --- a/samples/desktopapp/user_credentials.py +++ b/samples/desktopapp/user_credentials.py @@ -61,10 +61,10 @@ def main(project: str) -> None: WHERE name = 'William' GROUP BY name; """ - query_job = client.query(query_string) + results = client.query_and_wait(query_string) # Print the results. - for row in query_job.result(): # Wait for the job to complete. + for row in results: # Wait for the job to complete. print("{}: {}".format(row["name"], row["total"])) # [END bigquery_auth_user_query] From 4ba434287a0a25f027e3b63a80f8881a9b16723e Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 23 Jan 2024 14:08:58 -0600 Subject: [PATCH 09/11] fix: `query_and_wait` now retains unknown query configuration `_properties` (#1793) * fix: `query_and_wait` now retains unknown query configuration `_properties` fix: raise `ValueError` in `query_and_wait` with wrong `job_config` type --- google/cloud/bigquery/_job_helpers.py | 24 +++++---- tests/unit/test__job_helpers.py | 75 +++++++++++++++++++++++---- 2 files changed, 79 insertions(+), 20 deletions(-) diff --git a/google/cloud/bigquery/_job_helpers.py b/google/cloud/bigquery/_job_helpers.py index 7356331b8..6debc377b 100644 --- a/google/cloud/bigquery/_job_helpers.py +++ b/google/cloud/bigquery/_job_helpers.py @@ -166,6 +166,14 @@ def do_query(): return future +def _validate_job_config(request_body: Dict[str, Any], invalid_key: str): + """Catch common mistakes, such as passing in a *JobConfig object of the + wrong type. + """ + if invalid_key in request_body: + raise ValueError(f"got unexpected key {repr(invalid_key)} in job_config") + + def _to_query_request( job_config: Optional[job.QueryJobConfig] = None, *, @@ -179,17 +187,15 @@ def _to_query_request( QueryRequest. If any configuration property is set that is not available in jobs.query, it will result in a server-side error. """ - request_body = {} - job_config_resource = job_config.to_api_repr() if job_config else {} - query_config_resource = job_config_resource.get("query", {}) + request_body = copy.copy(job_config.to_api_repr()) if job_config else {} - request_body.update(query_config_resource) + _validate_job_config(request_body, job.CopyJob._JOB_TYPE) + _validate_job_config(request_body, job.ExtractJob._JOB_TYPE) + _validate_job_config(request_body, job.LoadJob._JOB_TYPE) - # These keys are top level in job resource and query resource. - if "labels" in job_config_resource: - request_body["labels"] = job_config_resource["labels"] - if "dryRun" in job_config_resource: - request_body["dryRun"] = job_config_resource["dryRun"] + # Move query.* properties to top-level. + query_config_resource = request_body.pop("query", {}) + request_body.update(query_config_resource) # Default to standard SQL. request_body.setdefault("useLegacySql", False) diff --git a/tests/unit/test__job_helpers.py b/tests/unit/test__job_helpers.py index f2fe32d94..404a546ff 100644 --- a/tests/unit/test__job_helpers.py +++ b/tests/unit/test__job_helpers.py @@ -23,6 +23,9 @@ from google.cloud.bigquery.client import Client from google.cloud.bigquery import _job_helpers +from google.cloud.bigquery.job import copy_ as job_copy +from google.cloud.bigquery.job import extract as job_extract +from google.cloud.bigquery.job import load as job_load from google.cloud.bigquery.job import query as job_query from google.cloud.bigquery.query import ConnectionProperty, ScalarQueryParameter @@ -57,9 +60,34 @@ def make_query_response( @pytest.mark.parametrize( ("job_config", "expected"), ( - (None, make_query_request()), - (job_query.QueryJobConfig(), make_query_request()), - ( + pytest.param( + None, + make_query_request(), + id="job_config=None-default-request", + ), + pytest.param( + job_query.QueryJobConfig(), + make_query_request(), + id="job_config=QueryJobConfig()-default-request", + ), + pytest.param( + job_query.QueryJobConfig.from_api_repr( + { + "unknownTopLevelProperty": "some-test-value", + "query": { + "unknownQueryProperty": "some-other-value", + }, + }, + ), + make_query_request( + { + "unknownTopLevelProperty": "some-test-value", + "unknownQueryProperty": "some-other-value", + } + ), + id="job_config-with-unknown-properties-includes-all-properties-in-request", + ), + pytest.param( job_query.QueryJobConfig(default_dataset="my-project.my_dataset"), make_query_request( { @@ -69,17 +97,24 @@ def make_query_response( } } ), + id="job_config-with-default_dataset", ), - (job_query.QueryJobConfig(dry_run=True), make_query_request({"dryRun": True})), - ( + pytest.param( + job_query.QueryJobConfig(dry_run=True), + make_query_request({"dryRun": True}), + id="job_config-with-dry_run", + ), + pytest.param( job_query.QueryJobConfig(use_query_cache=False), make_query_request({"useQueryCache": False}), + id="job_config-with-use_query_cache", ), - ( + pytest.param( job_query.QueryJobConfig(use_legacy_sql=True), make_query_request({"useLegacySql": True}), + id="job_config-with-use_legacy_sql", ), - ( + pytest.param( job_query.QueryJobConfig( query_parameters=[ ScalarQueryParameter("named_param1", "STRING", "param-value"), @@ -103,8 +138,9 @@ def make_query_response( ], } ), + id="job_config-with-query_parameters-named", ), - ( + pytest.param( job_query.QueryJobConfig( query_parameters=[ ScalarQueryParameter(None, "STRING", "param-value"), @@ -126,8 +162,9 @@ def make_query_response( ], } ), + id="job_config-with-query_parameters-positional", ), - ( + pytest.param( job_query.QueryJobConfig( connection_properties=[ ConnectionProperty(key="time_zone", value="America/Chicago"), @@ -142,14 +179,17 @@ def make_query_response( ] } ), + id="job_config-with-connection_properties", ), - ( + pytest.param( job_query.QueryJobConfig(labels={"abc": "def"}), make_query_request({"labels": {"abc": "def"}}), + id="job_config-with-labels", ), - ( + pytest.param( job_query.QueryJobConfig(maximum_bytes_billed=987654), make_query_request({"maximumBytesBilled": "987654"}), + id="job_config-with-maximum_bytes_billed", ), ), ) @@ -159,6 +199,19 @@ def test__to_query_request(job_config, expected): assert result == expected +@pytest.mark.parametrize( + ("job_config", "invalid_key"), + ( + pytest.param(job_copy.CopyJobConfig(), "copy", id="copy"), + pytest.param(job_extract.ExtractJobConfig(), "extract", id="extract"), + pytest.param(job_load.LoadJobConfig(), "load", id="load"), + ), +) +def test__to_query_request_raises_for_invalid_config(job_config, invalid_key): + with pytest.raises(ValueError, match=f"{repr(invalid_key)} in job_config"): + _job_helpers._to_query_request(job_config, query="SELECT 1") + + def test__to_query_job_defaults(): mock_client = mock.create_autospec(Client) response = make_query_response( From 6559dde1568b2d07c1e0a142194f2b370efb8983 Mon Sep 17 00:00:00 2001 From: Gaurang Shah Date: Tue, 23 Jan 2024 19:03:36 -0500 Subject: [PATCH 10/11] feature: add query location for bigquery magic (#1771) Co-authored-by: Lingqing Gan --- google/cloud/bigquery/magics/magics.py | 11 +++++++++++ tests/unit/test_magics.py | 18 ++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/google/cloud/bigquery/magics/magics.py b/google/cloud/bigquery/magics/magics.py index 2a3583c66..b7c685d9a 100644 --- a/google/cloud/bigquery/magics/magics.py +++ b/google/cloud/bigquery/magics/magics.py @@ -508,6 +508,15 @@ def _create_dataset_if_necessary(client, dataset_id): "Defaults to use tqdm_notebook. Install the ``tqdm`` package to use this feature." ), ) +@magic_arguments.argument( + "--location", + type=str, + default=None, + help=( + "Set the location to execute query." + "Defaults to location set in query setting in console." + ), +) def _cell_magic(line, query): """Underlying function for bigquery cell magic @@ -551,6 +560,7 @@ def _cell_magic(line, query): category=DeprecationWarning, ) use_bqstorage_api = not args.use_rest_api + location = args.location params = [] if params_option_value: @@ -579,6 +589,7 @@ def _cell_magic(line, query): default_query_job_config=context.default_query_job_config, client_info=client_info.ClientInfo(user_agent=IPYTHON_USER_AGENT), client_options=bigquery_client_options, + location=location, ) if context._connection: client._connection = context._connection diff --git a/tests/unit/test_magics.py b/tests/unit/test_magics.py index b03894095..1511cba9c 100644 --- a/tests/unit/test_magics.py +++ b/tests/unit/test_magics.py @@ -2053,3 +2053,21 @@ def test_bigquery_magic_create_dataset_fails(): ) assert close_transports.called + + +@pytest.mark.usefixtures("ipython_interactive") +def test_bigquery_magic_with_location(): + ip = IPython.get_ipython() + ip.extension_manager.load_extension("google.cloud.bigquery") + magics.context.credentials = mock.create_autospec( + google.auth.credentials.Credentials, instance=True + ) + + run_query_patch = mock.patch( + "google.cloud.bigquery.magics.magics._run_query", autospec=True + ) + with run_query_patch as run_query_mock: + ip.run_cell_magic("bigquery", "--location=us-east1", "SELECT 17 AS num") + + client_options_used = run_query_mock.call_args_list[0][0][0] + assert client_options_used.location == "us-east1" From 17e9d06a1d1c2048ea9ec3c627a9afe654584a6b Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 24 Jan 2024 13:39:09 -0600 Subject: [PATCH 11/11] chore(main): release 3.17.0 (#1780) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 22 ++++++++++++++++++++++ google/cloud/bigquery/version.py | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25c4ca1e5..bb916158d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,28 @@ [1]: https://pypi.org/project/google-cloud-bigquery/#history +## [3.17.0](https://github.com/googleapis/python-bigquery/compare/v3.16.0...v3.17.0) (2024-01-24) + + +### Features + +* Support universe resolution ([#1774](https://github.com/googleapis/python-bigquery/issues/1774)) ([0b5c1d5](https://github.com/googleapis/python-bigquery/commit/0b5c1d597cdec3a05a16fb935595f773c5840bd4)) + + +### Bug Fixes + +* `query_and_wait` now retains unknown query configuration `_properties` ([#1793](https://github.com/googleapis/python-bigquery/issues/1793)) ([4ba4342](https://github.com/googleapis/python-bigquery/commit/4ba434287a0a25f027e3b63a80f8881a9b16723e)) +* Raise `ValueError` in `query_and_wait` with wrong `job_config` type ([4ba4342](https://github.com/googleapis/python-bigquery/commit/4ba434287a0a25f027e3b63a80f8881a9b16723e)) + + +### Documentation + +* Remove unused query code sample ([#1769](https://github.com/googleapis/python-bigquery/issues/1769)) ([1f96439](https://github.com/googleapis/python-bigquery/commit/1f96439b3dbd27f11be5e2af84f290ec6094d0a4)) +* Update `snippets.py` to use `query_and_wait` ([#1773](https://github.com/googleapis/python-bigquery/issues/1773)) ([d90602d](https://github.com/googleapis/python-bigquery/commit/d90602de87e58b665cb974401a327a640805822f)) +* Update multiple samples to change query to query_and_wait ([#1784](https://github.com/googleapis/python-bigquery/issues/1784)) ([d1161dd](https://github.com/googleapis/python-bigquery/commit/d1161dddde41a7d35b30033ccbf6984a5de640bd)) +* Update the query with no cache sample to use query_and_wait API ([#1770](https://github.com/googleapis/python-bigquery/issues/1770)) ([955a4cd](https://github.com/googleapis/python-bigquery/commit/955a4cd99e21cbca1b2f9c1dc6aa3fd8070cd61f)) +* Updates `query` to `query and wait` in samples/desktopapp/user_credentials.py ([#1787](https://github.com/googleapis/python-bigquery/issues/1787)) ([89f1299](https://github.com/googleapis/python-bigquery/commit/89f1299b3164b51fb0f29bc600a34ded59c10682)) + ## [3.16.0](https://github.com/googleapis/python-bigquery/compare/v3.15.0...v3.16.0) (2024-01-12) diff --git a/google/cloud/bigquery/version.py b/google/cloud/bigquery/version.py index a3de40375..9f62912a4 100644 --- a/google/cloud/bigquery/version.py +++ b/google/cloud/bigquery/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "3.16.0" +__version__ = "3.17.0"