8000 tests: allow prerelease dependency versions under Python 3.9 by tseaver · Pull Request #479 · googleapis/python-spanner · GitHub
[go: up one dir, main page]
Skip to content

tests: allow prerelease dependency versions under Python 3.9 #479

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Aug 12, 2021
Merged

tests: allow prerelease dependency versions under Python 3.9 #479

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
b1d4112
tests: allow prerelease dependency versions under Python 3.9
tseaver Aug 6, 2021
0e05f4e
chore: debug installed OT versions on CI
tseaver Aug 10, 2021
ef8561a
ci: force upgrade of pip to fix OT mangling
tseaver Aug 10, 2021
1f4e858
ci: harden samples against 429 in instance creation / deletion
tseaver Aug 10, 2021
b04bf32
ci: further harden samples against 429 in instance deletion
tseaver Aug 10, 2021
b5a0bf2
Merge branch 'master' into tests-allow-prerelease-deps-python39
tseaver Aug 11, 2021
78615bf
tests: bump operation timeout for snippets to 240 seconds
tseaver Aug 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions docs/snapshot-usage.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,7 @@ reads as of a given timestamp:
.. code:: python

import datetime
from pytz import UTC
TIMESTAMP = datetime.datetime.utcnow().replace(tzinfo=UTC)
TIMESTAMP = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc)

with database.snapshot(read_timestamp=TIMESTAMP) as snapshot:
...
Expand Down
4 changes: 4 additions & 0 deletions noxfile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,11 @@ def default(session):
*session.posargs,
)

# XXX Work around Kokoro image's older pip, which borks the OT install.
session.run("pip", "install", "--upgrade", "pip")
session.install("-e", ".[tracing]", "-c", constraints_path)
# XXX: Dump installed versions to debug OT issue
session.run("pip", "list")

# Run py.test against the unit tests with OpenTelemetry.
session.run(
Expand Down
95 changes: 65 additions & 30 deletions owlbot.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,16 @@

common = gcp.CommonTemplates()

# This is a customized version of the s.get_staging_dirs() function from synthtool to
# cater for copying 3 different folders from googleapis-gen
# which are spanner, spanner/admin/instance and spanner/admin/database.
# Source https://github.com/googleapis/synthtool/blob/master/synthtool/transforms.py#L280

def get_staging_dirs(
default_version: Optional[str] = None, sub_directory: Optional[str] = None
# This is a customized version of the s.get_staging_dirs() function
# from synthtool to # cater for copying 3 different folders from
# googleapis-gen:
# spanner, spanner/admin/instance and spanner/admin/database.
# Source:
# https://github.com/googleapis/synthtool/blob/master/synthtool/transforms.py#L280
default_version: Optional[str] = None,
sub_directory: Optional[str] = None,
) -> List[Path]:
"""Returns the list of directories, one per version, copied from
https://github.com/googleapis/googleapis-gen. Will return in lexical sorting
Expand Down Expand Up @@ -63,46 +67,69 @@ def get_staging_dirs(
else:
return []


spanner_default_version = "v1"
spanner_admin_instance_default_version = "v1"
spanner_admin_database_default_version = "v1"

for library in get_staging_dirs(spanner_default_version, "spanner"):
# Work around gapic generator bug https://github.com/googleapis/gapic-generator-python/issues/902
s.replace(library / f"google/cloud/spanner_{library.name}/types/transaction.py",
r""".
s.replace(
library / f"google/cloud/spanner_{library.name}/types/transaction.py",
r""".
Attributes:""",
r""".\n
r""".\n
Attributes:""",
)

# Work around gapic generator bug https://github.com/googleapis/gapic-generator-python/issues/902
s.replace(library / f"google/cloud/spanner_{library.name}/types/transaction.py",
r""".
s.replace(
library / f"google/cloud/spanner_{library.name}/types/transaction.py",
r""".
Attributes:""",
r""".\n
r""".\n
Attributes:""",
)

# Remove headings from docstring. Requested change upstream in cl/377290854 due to https://google.aip.dev/192#formatting.
s.replace(library / f"google/cloud/spanner_{library.name}/types/transaction.py",
s.replace(
library / f"google/cloud/spanner_{library.name}/types/transaction.py",
"""\n ==.*?==\n""",
":",
)

# Remove headings from docstring. Requested change upstream in cl/377290854 due to https://google.aip.dev/192#formatting.
s.replace(library / f"google/cloud/spanner_{library.name}/types/transaction.py",
s.replace(
library / f"google/cloud/spanner_{library.name}/types/transaction.py",
"""\n --.*?--\n""",
":",
)

s.move(library, excludes=["google/cloud/spanner/**", "*.*", "docs/index.rst", "google/cloud/spanner_v1/__init__.py"])
s.move(
library,
excludes=[
"google/cloud/spanner/**",
"*.*",
"docs/index.rst",
"google/cloud/spanner_v1/__init__.py",
],
)

for library in get_staging_dirs(spanner_admin_instance_default_version, "spanner_admin_instance"):
s.move(library, excludes=["google/cloud/spanner_admin_instance/**", "*.*", "docs/index.rst"])
for library in get_staging_dirs(
spanner_admin_instance_default_version, "spanner_admin_instance"
):
s.move(
library,
excludes=["google/cloud/spanner_admin_instance/**", "*.*", "docs/index.rst"],
)

for library in get_staging_dirs(spanner_admin_database_default_version, "spanner_admin_database"):
s.move(library, excludes=["google/cloud/spanner_admin_database/**", "*.*", "docs/index.rst"])
for library in get_staging_dirs(
spanner_admin_database_default_version, "spanner_admin_database"
):
s.move(
library,
excludes=["google/cloud/spanner_admin_database/**", "*.*", "docs/index.rst"],
)

s.remove_staging_dirs()

Expand All @@ -116,9 +143,11 @@ def get_staging_dirs(
s.replace(
".kokoro/build.sh",
"# Remove old nox",
"# Set up creating a new instance for each system test run\n"
"export GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE=true\n"
"\n\g<0>",
"""\
# Set up creating a new instance for each system test run
export GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE=true

# Remove old nox""",
)

# Update samples folder in CONTRIBUTING.rst
Expand All @@ -134,15 +163,21 @@ def get_staging_dirs(
# Customize noxfile.py
# ----------------------------------------------------------------------------


def place_before(path, text, *before_text, escape=None):
replacement = "\n".join(before_text) + "\n" + text
if escape:
for c in escape:
text = text.replace(c, '\\' + c)
text = text.replace(c, "\\" + c)
s.replace([path], text, replacement)


open_telemetry_test = """
# XXX Work around Kokoro image's older pip, which borks the OT install.
session.run("pip", "install", "--upgrade", "pip")
session.install("-e", ".[tracing]", "-c", constraints_path)
# XXX: Dump installed versions to debug OT issue
session.run("pip", "list")

# Run py.test against the unit tests with OpenTelemetry.
session.run(
Expand All @@ -164,10 +199,10 @@ def place_before(path, text, *before_text, escape=None):
"noxfile.py",
"@nox.session(python=UNIT_TEST_PYTHON_VERSIONS)",
open_telemetry_test,
escape="()"
escape="()",
)

skip_tests_if_env_var_not_set ="""# Sanity check: Only run tests if the environment variable is set.
skip_tests_if_env_var_not_set = """# Sanity check: Only run tests if the environment variable is set.
if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", "") and not os.environ.get(
"SPANNER_EMULATOR_HOST", ""
):
Expand All @@ -180,7 +215,7 @@ def place_before(path, text, *before_text, escape=None):
"noxfile.py",
"# Install pyopenssl for mTLS testing.",
skip_tests_if_env_var_not_set,
escape="()"
escape="()",
)

s.replace(
685C Expand All @@ -190,25 +225,25 @@ def place_before(path, text, *before_text, escape=None):
"--cov=tests/unit",""",
"""\"--cov=google.cloud.spanner",
"--cov=google.cloud",
"--cov=tests.unit","""
"--cov=tests.unit",""",
)

s.replace(
"noxfile.py",
"""session.install\("-e", "."\)""",
"""session.install("-e", ".[tracing]")"""
r"""session.install\("-e", "."\)""",
"""session.install("-e", ".[tracing]")""",
)

s.replace(
"noxfile.py",
"""# Install all test dependencies, then install this package into the
r"""# Install all test dependencies, then install this package into the
# virtualenv's dist-packages.
session.install\("mock", "pytest", "google-cloud-testutils", "-c", constraints_path\)
session.install\("-e", ".", "-c", constraints_path\)""",
"""# Install all test dependencies, then install this package into the
# virtualenv's dist-packages.
session.install("mock", "pytest", "google-cloud-testutils", "-c", constraints_path)
session.install("-e", ".[tracing]", "-c", constraints_path)"""
session.install("-e", ".[tracing]", "-c", constraints_path)""",
)

s.shell.run(["nox", "-s", "blacken"], hide_output=False)
6 changes: 3 additions & 3 deletions
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
import pytest
from test_utils import retry

retry_429 = retry.RetryErrors(exceptions.ResourceExhausted, delay=15)


@pytest.fixture(scope="module")
def sample_name():
Expand All @@ -47,7 +49,7 @@ def scrub_instance_ignore_not_found(to_scrub):
for backup_pb in to_scrub.list_backups():
backup.Backup.from_pb(backup_pb, to_scrub).delete()

to_scrub.delete()
retry_429(to_scrub.delete)()
except exceptions.NotFound:
pass

Expand Down Expand Up @@ -107,7 +109,6 @@ def sample_instance(
"created": str(int(time.time())),
},
)
retry_429 = retry.RetryErrors(exceptions.ResourceExhausted, delay=15)
op = retry_429(sample_instance.create)()
op.result(120) # block until completion

Expand Down Expand Up @@ -143,7 +144,6 @@ def multi_region_instance(
"created": str(int(time.time()))
},
)
retry_429 = retry.RetryErrors(exceptions.ResourceExhausted, delay=15)
op = retry_429(multi_region_instance.create)()
op.result(120) # block until completion

Expand Down
28 changes: 15 additions & 13 deletions samples/samples/snippets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@
from google.cloud import spanner
from google.cloud.spanner_v1 import param_types

OPERATION_TIMEOUT_SECONDS = 240


# [START spanner_create_instance]
def create_instance(instance_id):
Expand All @@ -55,7 +57,7 @@ def create_instance(instance_id):
operation = instance.create()

print("Waiting for operation to complete...")
operation.result(120)
operation.result(OPERATION_TIMEOUT_SECONDS)

print("Created instance {}".format(instance_id))

Expand Down Expand Up @@ -87,7 +89,7 @@ def create_instance_with_processing_units(instance_id, processing_units):
operation = instance.create()

print("Waiting for operation to complete...")
operation.result(120)
operation.result(OPERATION_TIMEOUT_SECONDS)

print("Created instance {} with {} processing units".format(
instance_id, instance.processing_units))
Expand Down Expand Up @@ -170,7 +172,7 @@ def create_database(instance_id, database_id):
operation = database.create()

print("Waiting for operation to complete...")
operation.result(120)
operation.result(OPERATION_TIMEOUT_SECONDS)

print("Created database {} on instance {}".format(database_id, instance_id))

Expand Down Expand Up @@ -206,7 +208,7 @@ def create_database_with_encryption_key(instance_id, database_id, kms_key_name):
operation = database.create()

print("Waiting for operation to complete...")
operation.result(120)
operation.result(OPERATION_TIMEOUT_SECONDS)

print("Database {} created with encryption key {}".format(
database.name, database.encryption_config.kms_key_name))
Expand Down Expand Up @@ -245,7 +247,7 @@ def create_database_with_default_leader(
operation = database.create()

print("Waiting for operation to complete...")
operation.result(120)
operation.result(OPERATION_TIMEOUT_SECONDS)

database.reload()

Expand All @@ -271,7 +273,7 @@ def update_database_with_default_leader(

operation = database.update_ddl(["ALTER DATABASE {}"
" SET OPTIONS (default_leader = '{}')".format(database_id, default_leader)])
operation.result(120)
operation.result(OPERATION_TIMEOUT_SECONDS)

database.reload()

Expand Down Expand Up @@ -499,7 +501,7 @@ def add_index(instance_id, database_id):
)

print("Waiting for operation to complete...")
operation.result(120)
operation.result(OPERATION_TIMEOUT_SECONDS)

print("Added the AlbumsByAlbumTitle index.")

Expand Down Expand Up @@ -598,7 +600,7 @@ def add_storing_index(instance_id, database_id):
)

print("Waiting for operation to complete...")
operation.result(120)
operation.result(OPERATION_TIMEOUT_SECONDS)

print("Added the AlbumsByAlbumTitle2 index.")

Expand Down Expand Up @@ -651,7 +653,7 @@ def add_column(instance_id, database_id):
)

print("Waiting for operation to complete...")
operation.result(120)
operation.result(OPERATION_TIMEOUT_SECONDS)

print("Added the MarketingBudget column.")

Expand Down Expand Up @@ -816,7 +818,7 @@ def create_table_with_timestamp(instance_id, database_id):
)

print("Waiting for operation to complete...")
operation.result(120)
operation.result(OPERATION_TIMEOUT_SECONDS)

print(
"Created Performances table on database {} on instance {}".format(
Expand Down Expand Up @@ -871,7 +873,7 @@ def add_timestamp_column(instance_id, database_id):
)

print("Waiting for operation to complete...")
operation.result(120)
operation.result(OPERATION_TIMEOUT_SECONDS)

print(
'Altered table "Albums" on database {} on instance {}.'.format(
Expand Down Expand Up @@ -964,7 +966,7 @@ def add_numeric_column(instance_id, database_id):
operation = database.update_ddl(["ALTER TABLE Venues ADD COLUMN Revenue NUMERIC"])

print("Waiting for operation to complete...")
operation.result(120)
operation.result(OPERATION_TIMEOUT_SECONDS)

print(
'Altered table "Venues" on database {} on instance {}.'.format(
Expand Down Expand Up @@ -1564,7 +1566,7 @@ def create_table_with_datatypes(instance_id, database_id):
)

print("Waiting for operation to complete...")
operation.result(120)
operation.result(OPERATION_TIMEOUT_SECONDS)

print(
"Created Venues table on database {} on instance {}".format(
Expand Down
Loading
0