8000 feat: define Datastore -> Firestore in Datastore mode migration long running operation metadata by gcf-owl-bot[bot] · Pull Request #261 · googleapis/python-datastore · GitHub
[go: up one dir, main page]

Skip to content

feat: define Datastore -> Firestore in Datastore mode migration long running operation metadata #261

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions google/cloud/datastore_admin_v1/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

from .types.datastore_admin import CommonMetadata
from .types.datastore_admin import CreateIndexRequest
from .types.datastore_admin import DatastoreFirestoreMigrationMetadata
from .types.datastore_admin import DeleteIndexRequest
from .types.datastore_admin import EntityFilter
from .types.datastore_admin import ExportEntitiesMetadata
Expand All @@ -33,12 +34,17 @@
from .types.datastore_admin import Progress
from .types.datastore_admin import OperationType
from .types.index import Index
from .types.migration import MigrationProgressEvent
from .types.migration import MigrationStateEvent
from .types.migration import MigrationState
from .types.migration import MigrationStep

__all__ = (
"DatastoreAdminAsyncClient",
"CommonMetadata",
"CreateIndexRequest",
"DatastoreAdminClient",
"DatastoreFirestoreMigrationMetadata",
"DeleteIndexRequest",
"EntityFilter",
"ExportEntitiesMetadata",
Expand All @@ -51,6 +57,10 @@
"IndexOperationMetadata",
"ListIndexesRequest",
"ListIndexesResponse",
"MigrationProgressEvent",
"MigrationState",
"MigrationStateEvent",
"MigrationStep",
"OperationType",
"Progress",
)
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,12 @@
except AttributeError: # pragma: NO COVER
OptionalRetry = Union[retries.Retry, object] # type: ignore

from google.api_core import operation
from google.api_core import operation_async
from google.api_core import operation # type: ignore
from google.api_core import operation_async # type: ignore
from google.cloud.datastore_admin_v1.services.datastore_admin import pagers
from google.cloud.datastore_admin_v1.types import datastore_admin
from google.cloud.datastore_admin_v1.types import index
from google.protobuf import empty_pb2
from google.protobuf import empty_pb2 # type: ignore
from .transports.base import DatastoreAdminTransport, DEFAULT_CLIENT_INFO
from .transports.grpc_asyncio import DatastoreAdminGrpcAsyncIOTransport
from .client import DatastoreAdminClient
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,12 @@
except AttributeError: # pragma: NO COVER
OptionalRetry = Union[retries.Retry, object] # type: ignore

from google.api_core import operation
from google.api_core import operation_async
from google.api_core import operation # type: ignore
from google.api_core import operation_async # type: ignore
from google.cloud.datastore_admin_v1.services.datastore_admin import pagers
from google.cloud.datastore_admin_v1.types import datastore_admin
from google.cloud.datastore_admin_v1.types import index
from google.protobuf import empty_pb2
from google.protobuf import empty_pb2 # type: ignore
from .transports.base import DatastoreAdminTransport, DEFAULT_CLIENT_INFO
from .transports.grpc import DatastoreAdminGrpcTransport
from .transports.grpc_asyncio import DatastoreAdminGrpcAsyncIOTransport
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@

from google.cloud.datastore_admin_v1.types import datastore_admin
from google.cloud.datastore_admin_v1.types import index
from google.longrunning import operations_pb2
from google.longrunning import operations_pb2 # type: ignore

try:
DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo(
Expand Down Expand Up @@ -107,7 +107,6 @@ def __init__(
credentials, _ = google.auth.load_credentials_from_file(
credentials_file, **scopes_kwargs, quota_project_id=quota_project_id
)

elif credentials is None:
credentials, _ = google.auth.default(
**scopes_kwargs, quota_project_id=quota_project_id
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,11 @@
from google.auth import credentials as ga_credentials # type: ignore
from google.auth.transport.grpc import SslCredentials # type: ignore

import grpc
import grpc # type: ignore

from google.cloud.datastore_admin_v1.types import datastore_admin
from google.cloud.datastore_admin_v1.types import index
from google.longrunning import operations_pb2
from google.longrunning import operations_pb2 # type: ignore
from .base import DatastoreAdminTransport, DEFAULT_CLIENT_INFO


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@
from google.auth import credentials as ga_credentials # type: ignore
from google.auth.transport.grpc import SslCredentials # type: ignore

import grpc
from grpc.experimental import aio
import grpc # type: ignore
from grpc.experimental import aio # type: ignore

from google.cloud.datastore_admin_v1.types import datastore_admin
from google.cloud.datastore_admin_v1.types import index
from google.longrunning import operations_pb2
from google.longrunning import operations_pb2 # type: ignore
from .base import DatastoreAdminTransport, DEFAULT_CLIENT_INFO
from .grpc import DatastoreAdminGrpcTransport

Expand Down
12 changes: 12 additions & 0 deletions google/cloud/datastore_admin_v1/types/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from .datastore_admin import (
CommonMetadata,
CreateIndexRequest,
DatastoreFirestoreMigrationMetadata,
DeleteIndexRequest,
EntityFilter,
ExportEntitiesMetadata,
Expand All @@ -31,10 +32,17 @@
OperationType,
)
from .index import Index
from .migration import (
MigrationProgressEvent,
MigrationStateEvent,
MigrationState,
MigrationStep,
)

__all__ = (
"CommonMetadata",
"CreateIndexRequest",
"DatastoreFirestoreMigrationMetadata",
"DeleteIndexRequest",
"EntityFilter",
"ExportEntitiesMetadata",
Expand All @@ -49,4 +57,8 @@
"Progress",
"OperationType",
"Index",
"MigrationProgressEvent",
"MigrationStateEvent",
"MigrationState",
"MigrationStep",
)
29 changes: 27 additions & 2 deletions google/cloud/datastore_admin_v1/types/datastore_admin.py
10000
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
import proto
import proto # type: ignore

from google.cloud.datastore_admin_v1.types import index as gda_index
from google.protobuf import timestamp_pb2
from google.cloud.datastore_admin_v1.types import migration
from google.protobuf import timestamp_pb2 # type: ignore


__protobuf__ = proto.module(
Expand All @@ -37,6 +38,7 @@
"ListIndexesRequest",
"ListIndexesResponse",
"IndexOperationMetadata",
"DatastoreFirestoreMigrationMetadata",
},
)

Expand Down Expand Up @@ -410,4 +412,27 @@ class IndexOperationMetadata(proto.Message):
index_id = proto.Field(proto.STRING, number=3,)


class DatastoreFirestoreMigrationMetadata(proto.Message):
r"""Metadata for Datastore to Firestore migration operations.

The DatastoreFirestoreMigration operation is not started by the
end-user via an explicit "creation" method. This is an intentional
deviation from the LRO design pattern.

This singleton resource can be accessed at:
``projects/{project_id}/datastore-firestore-migration``

Attributes:
migration_state (google.cloud.datastore_admin_v1.types.MigrationState):
The current state of migration from Cloud
Datastore to Cloud Firestore in Datastore mode.
migration_step (google.cloud.datastore_admin_v1.types.MigrationStep):
The current step of migration from Cloud
Datastore to Cloud Firestore in Datastore mode.
"""

migration_state = proto.Field(proto.ENUM, number=1, enum=migration.MigrationState,)
migration_step = proto.Field(proto.ENUM, number=2, enum=migration.MigrationStep,)


__all__ = tuple(sorted(__protobuf__.manifest))
2 changes: 1 addition & 1 deletion google/cloud/datastore_admin_v1/types/index.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
import proto
import proto # type: ignore


__protobuf__ = proto.module(package="google.datastore.admin.v1", manifest={"Index",},)
Expand Down
135 changes: 135 additions & 0 deletions google/cloud/datastore_admin_v1/types/migration.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
# -*- coding: utf-8 -*-
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import proto # type: ignore


__protobuf__ = proto.module(
package="google.datastore.admin.v1",
manifest={
"MigrationState",
"MigrationStep",
"MigrationStateEvent",
"MigrationProgressEvent",
},
)


class MigrationState(proto.Enum):
r"""States for a migration."""
MIGRATION_STATE_UNSPECIFIED = 0
RUNNING = 1
PAUSED = 2
COMPLETE = 3


class MigrationStep(proto.Enum):
r"""Steps in a migration."""
MIGRATION_STEP_UNSPECIFIED = 0
PREPARE = 6
START = 1
APPLY_WRITES_SYNCHRONOUSLY = 7
COPY_AND_VERIFY = 2
REDIRECT_EVENTUALLY_CONSISTENT_READS = 3
REDIRECT_STRONGLY_CONSISTENT_READS = 4
REDIRECT_WRITES = 5


class MigrationStateEvent(proto.Message):
r"""An event signifying a change in state of a `migration from Cloud
Datastore to Cloud Firestore in Datastore
mode <https://cloud.google.com/datastore/docs/upgrade-to-firestore>`__.

Attributes:
state (google.cloud.datastore_admin_v1.types.MigrationState):
The new state of the migration.
"""

state = proto.Field(proto.ENUM, number=1, enum="MigrationState",)


class MigrationProgressEvent(proto.Message):
r"""An event signifying the start of a new step in a `migration from
Cloud Datastore to Cloud Firestore in Datastore
mode <https://cloud.google.com/datastore/docs/upgrade-to-firestore>`__.

This message has `oneof`_ fields (mutually exclusive fields).
For each oneof, at most one member field can be set at the same time.
Setting any member of the oneof automatically clears all other
members.

.. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields

Attributes:
step (google.cloud.datastore_admin_v1.types.MigrationStep):
The step that is starting.

An event with step set to ``START`` indicates that the
migration has been reverted back to the initial
pre-migration state.
prepare_step_details (google.cloud.datastore_admin_v1.types.MigrationProgressEvent.PrepareStepDetails):
Details for the ``PREPARE`` step.

This field is a member of `oneof`_ ``step_details``.
redirect_writes_step_details (google.cloud.datastore_admin_v1.types.MigrationProgressEvent.RedirectWritesStepDetails):
Details for the ``REDIRECT_WRITES`` step.

This field is a member of `oneof`_ ``step_details``.
"""

class ConcurrencyMode(proto.Enum):
r"""Concurrency modes for transactions in Cloud Firestore."""
CONCURRENCY_MODE_UNSPECIFIED = 0
PESSIMISTIC = 1
OPTIMISTIC = 2

class PrepareStepDetails(proto.Message):
r"""Details for the ``PREPARE`` step.

Attributes:
concurrency_mode (google.cloud.datastore_admin_v1.types.MigrationProgressEvent.ConcurrencyMode):
The concurrency mode this database will use when it reaches
the ``REDIRECT_WRITES`` step.
"""

concurrency_mode = proto.Field(
proto.ENUM, number=1, enum="MigrationProgressEvent.ConcurrencyMode",
)

class RedirectWritesStepDetails(proto.Message):
r"""Details for the ``REDIRECT_WRITES`` step.

Attributes:
concurrency_mode (google.cloud.datastore_admin_v1.types.MigrationProgressEvent.ConcurrencyMode):
Ths concurrency mode for this database.
"""

concurrency_mode = proto.Field(
proto.ENUM, number=1, enum="MigrationProgressEvent.ConcurrencyMode",
)

step = proto.Field(proto.ENUM, number=1, enum="MigrationStep",)
prepare_step_details = proto.Field(
proto.MESSAGE, number=2, oneof="step_details", message=PrepareStepDetails,
)
redirect_writes_step_details = proto.Field(
proto.MESSAGE,
number=3,
oneof="step_details",
message=RedirectWritesStepDetails,
)


__all__ = tuple(sorted(__protobuf__.manifest))
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,6 @@ def __init__(
credentials, _ = google.auth.load_credentials_from_file(
credentials_file, **scopes_kwargs, quota_project_id=quota_project_id
)

elif credentials is None:
credentials, _ = google.auth.default(
**scopes_kwargs, quota_project_id=quota_project_id
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
from google.auth import credentials as ga_credentials # type: ignore
from google.auth.transport.grpc import SslCredentials # type: ignore

import grpc
import grpc # type: ignore

from google.cloud.datastore_v1.types import datastore
from .base import DatastoreTransport, DEFAULT_CLIENT_INFO
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@
from google.auth import credentials as ga_credentials # type: ignore
from google.auth.transport.grpc import SslCredentials # type: ignore

import grpc
from grpc.experimental import aio
import grpc # type: ignore
from grpc.experimental import aio # type: ignore

from google.cloud.datastore_v1.types import datastore
from .base import DatastoreTransport, DEFAULT_CLIENT_INFO
Expand Down
2 changes: 1 addition & 1 deletion google/cloud/datastore_v1/types/datastore.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
import proto
import proto # type: ignore

from google.cloud.datastore_v1.types import entity
from google.cloud.datastore_v1.types import query as gd_query
Expand Down
8 changes: 4 additions & 4 deletions google/cloud/datastore_v1/types/entity.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
import proto
import proto # type: ignore

from google.protobuf import struct_pb2
from google.protobuf import timestamp_pb2
from google.type import latlng_pb2
from google.protobuf import struct_pb2 # type: ignore
from google.protobuf import timestamp_pb2 # type: ignore
from google.type import latlng_pb2 # type: ignore


__protobuf__ = proto.module(
Expand Down
Loading
0