8000 feat: add support for Dataproc metric configuration by gcf-owl-bot[bot] · Pull Request #481 · googleapis/python-dataproc · GitHub
[go: up one dir, main page]
Skip to content
This repository was archived by the owner on Nov 29, 2023. It is now read-only.

feat: add support for Dataproc metric configuration #481

Merged
parthea merged 2 commits into from
Sep 21, 2022
Merged

feat: add support for Dataproc metric configuration #481

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0746f70
feat: add support for Dataproc metric configuration
gcf-owl-bot[bot] Sep 21, 2022
a2538b0
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Sep 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions google/cloud/dataproc/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@
from google.cloud.dataproc_v1.types.clusters import ClusterStatus
from google.cloud.dataproc_v1.types.clusters import ConfidentialInstanceConfig
from google.cloud.dataproc_v1.types.clusters import CreateClusterRequest
from google.cloud.dataproc_v1.types.clusters import DataprocMetricConfig
from google.cloud.dataproc_v1.types.clusters import DeleteClusterRequest
from google.cloud.dataproc_v1.types.clusters import DiagnoseClusterRequest
from google.cloud.dataproc_v1.types.clusters import DiagnoseClusterResults
Expand All @@ -106,6 +107,7 @@
from google.cloud.dataproc_v1.types.clusters import ListClustersResponse
from google.cloud.dataproc_v1.types.clusters import ManagedGroupConfig
from google.cloud.dataproc_v1.types.clusters import MetastoreConfig
from google.cloud.dataproc_v1.types.clusters import Metric
from google.cloud.dataproc_v1.types.clusters import NodeGroupAffinity
from google.cloud.dataproc_v1.types.clusters import NodeInitializationAction
from google.cloud.dataproc_v1.types.clusters import ReservationAffinity
Expand Down Expand Up @@ -232,6 +234,7 @@
"ClusterStatus",
"ConfidentialInstanceConfig",
"CreateClusterRequest",
"DataprocMetricConfig",
"DeleteClusterRequest",
"DiagnoseClusterRequest",
"DiagnoseClusterResults",
Expand All @@ -248,6 +251,7 @@
"ListClustersResponse",
"ManagedGroupConfig",
"MetastoreConfig",
"Metric",
"NodeGroupAffinity",
"NodeInitializationAction",
"ReservationAffinity",
Expand Down
4 changes: 4 additions & 0 deletions google/cloud/dataproc_v1/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
from .types.clusters import ClusterStatus
from .types.clusters import ConfidentialInstanceConfig
from .types.clusters import CreateClusterRequest
from .types.clusters import DataprocMetricConfig
from .types.clusters import DeleteClusterRequest
from .types.clusters import DiagnoseClusterRequest
from .types.clusters import DiagnoseClusterResults
Expand All @@ -70,6 +71,7 @@
from .types.clusters import ListClustersResponse
from .types.clusters import ManagedGroupConfig
from .types.clusters import MetastoreConfig
from .types.clusters import Metric
from .types.clusters import NodeGroupAffinity
from .types.clusters import NodeInitializationAction
from .types.clusters import ReservationAffinity
Expand Down Expand Up @@ -174,6 +176,7 @@
"CreateBatchRequest",
"CreateClusterRequest",
"CreateWorkflowTemplateRequest",
"DataprocMetricConfig",
"DeleteAutoscalingPolicyRequest",
"DeleteBatchRequest",
"DeleteClusterRequest",
Expand Down Expand Up @@ -228,6 +231,7 @@
"ManagedCluster",
"ManagedGroupConfig",
"MetastoreConfig",
"Metric",
"NodeGroupAffinity",
"NodeInitializationAction",
"OrderedJob",
Expand Down
4 changes: 4 additions & 0 deletions google/cloud/dataproc_v1/types/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
ClusterStatus,
ConfidentialInstanceConfig,
CreateClusterRequest,
DataprocMetricConfig,
DeleteClusterRequest,
DiagnoseClusterRequest,
DiagnoseClusterResults,
Expand All @@ -63,6 +64,7 @@
ListClustersResponse,
ManagedGroupConfig,
MetastoreConfig,
Metric,
NodeGroupAffinity,
NodeInitializationAction,
ReservationAffinity,
Expand Down Expand Up @@ -174,6 +176,7 @@
"ClusterStatus",
"ConfidentialInstanceConfig",
"CreateClusterRequest",
"DataprocMetricConfig",
"DeleteClusterRequest",
"DiagnoseClusterRequest",
"DiagnoseClusterResults",
Expand All @@ -190,6 +193,7 @@
"ListClustersResponse",
"ManagedGroupConfig",
"MetastoreConfig",
"Metric",
"NodeGroupAffinity",
"NodeInitializationAction",
"ReservationAffinity",
Expand Down
58 changes: 58 additions & 0 deletions google/cloud/dataproc_v1/types/clusters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@
"SoftwareConfig",
"LifecycleConfig",
"MetastoreConfig",
"Metric",
"DataprocMetricConfig",
"ClusterMetrics",
"CreateClusterRequest",
"UpdateClusterRequest",
Expand Down Expand Up @@ -234,6 +236,8 @@ class ClusterConfig(proto.Message):
this cluster
metastore_config (google.cloud.dataproc_v1.types.MetastoreConfig):
Optional. Metastore configuration.
dataproc_metric_config (google.cloud.dataproc_v1.types.DataprocMetricConfig):
Optional. Dataproc metrics configuration.
"""

config_bucket = proto.Field(
Expand Down Expand Up @@ -304,6 +308,11 @@ class ClusterConfig(proto.Message):
number=20,
message="MetastoreConfig",
)
dataproc_metric_config = proto.Field(
proto.MESSAGE,
number=21,
message="DataprocMetricConfig",
)


class VirtualClusterConfig(proto.Message):
Expand Down Expand Up @@ -1331,6 +1340,55 @@ class MetastoreConfig(proto.Message):
)


class Metric(proto.Message):
r"""Specifies Dataproc OSS Metric.

Attributes:
metric_source (google.cloud.dataproc_v1.types.Metric.MetricSource):
Required. Specified source of metric
collection
metric_overrides (Sequence[str]):
Optional. The set of available OSS metrics to
collect from the metric source.
"""

class MetricSource(proto.Enum):
r""""""
METRIC_SOURCE_UNSPECIFIED = 0
MONITORING_AGENT_DEFAULTS = 1
HDFS = 2
SPARK = 3
YARN = 4
SPARK_HISTORY_SERVER = 5
HIVESERVER2 = 6

metric_source = proto.Field(
proto.ENUM,
number=1,
enum=MetricSource,
)
metric_overrides = proto.RepeatedField(
proto.STRING,
number=2,
)


class DataprocMetricConfig(proto.Message):
r"""Specifies a Dataproc metric config

Attributes:
metrics (Sequence[google.cloud.dataproc_v1.types.Metric]):
Configuration set of metrics to collect from
the cluster
"""

metrics = proto.RepeatedField(
proto.MESSAGE,
number=1,
message="Metric",
)


class ClusterMetrics(proto.Message):
r"""Contains cluster daemon metrics, such as HDFS and YARN stats.

Expand Down
0