8000 chore: updated docs and code sturcture for Dataproc metric configuration by gcf-owl-bot[bot] · Pull Request #485 · googleapis/python-dataproc · GitHub
[go: up one dir, main page]
Skip to content
This repository was archived by the owner on Nov 29, 2023. It is now read-only.

chore: updated docs and code sturcture for Dataproc metric configuration #485

Merged
parthea merged 2 commits into from
Sep 27, 2022
Merged

chore: updated docs and code sturcture for Dataproc metric configuration #485

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
38ff7ae
chore: updated docs and code sturcture for Dataproc metric configuration
gcf-owl-bot[bot] Sep 27, 2022
c5ec95f
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Sep 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions google/cloud/dataproc/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,6 @@
from google.cloud.dataproc_v1.types.clusters import ListClustersResponse
from google.cloud.dataproc_v1.types.clusters import ManagedGroupConfig
from google.cloud.dataproc_v1.types.clusters import MetastoreConfig
from google.cloud.dataproc_v1.types.clusters import Metric
from google.cloud.dataproc_v1.types.clusters import NodeGroupAffinity
from google.cloud.dataproc_v1.types.clusters import NodeInitializationAction
from google.cloud.dataproc_v1.types.clusters import ReservationAffinity
Expand Down Expand Up @@ -251,7 +250,6 @@
"ListClustersResponse",
"ManagedGroupConfig",
"MetastoreConfig",
"Metric",
"NodeGroupAffinity",
"NodeInitializationAction",
"ReservationAffinity",
Expand Down
2 changes: 0 additions & 2 deletions google/cloud/dataproc_v1/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,6 @@
from .types.clusters import ListClustersResponse
from .types.clusters import ManagedGroupConfig
from .types.clusters import MetastoreConfig
from .types.clusters import Metric
from .types.clusters import NodeGroupAffinity
from .types.clusters import NodeInitializationAction
from .types.clusters import ReservationAffinity
Expand Down Expand Up @@ -231,7 +230,6 @@
"ManagedCluster",
"ManagedGroupConfig",
"MetastoreConfig",
"Metric",
"NodeGroupAffinity",
"NodeInitializationAction",
"OrderedJob",
Expand Down
2 changes: 0 additions & 2 deletions google/cloud/dataproc_v1/types/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,6 @@
ListClustersResponse,
ManagedGroupConfig,
MetastoreConfig,
Metric,
NodeGroupAffinity,
NodeInitializationAction,
ReservationAffinity,
Expand Down Expand Up @@ -193,7 +192,6 @@
"ListClustersResponse",
"ManagedGroupConfig",
"MetastoreConfig",
"Metric",
"NodeGroupAffinity",
"NodeInitializationAction",
"ReservationAffinity",
Expand Down
94 changes: 63 additions & 31 deletions google/cloud/dataproc_v1/types/clusters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@
"SoftwareConfig",
"LifecycleConfig",
"MetastoreConfig",
"Metric",
"DataprocMetricConfig",
"ClusterMetrics",
"CreateClusterRequest",
Expand Down Expand Up @@ -237,7 +236,7 @@ class ClusterConfig(proto.Message):
metastore_config (google.cloud.dataproc_v1.types.MetastoreConfig):
Optional. Metastore configuration.
dataproc_metric_config (google.cloud.dataproc_v1.types.DataprocMetricConfig):
Optional. Dataproc metrics configuration.
Optional. The config for Dataproc metrics.
"""

config_bucket = proto.Field(
8000 Expand Down Expand Up @@ -310,7 +309,7 @@ class ClusterConfig(proto.Message):
)
dataproc_metric_config = proto.Field(
proto.MESSAGE,
number=21,
number=23,
message="DataprocMetricConfig",
)

Expand Down Expand Up @@ -1340,20 +1339,19 @@ class MetastoreConfig(proto.Message):
)


class Metric(proto.Message):
r"""Specifies Dataproc OSS Metric.
class DataprocMetricConfig(proto.Message):
r"""Dataproc metric config.

Attributes:
metric_source (google.cloud.dataproc_v1.types.Metric.MetricSource):
Required. Specified source of metric
collection
metric_overrides (Sequence[str]):
Optional. The set of available OSS metrics to
collect from the metric source.
metrics (Sequence[google.cloud.dataproc_v1.types.DataprocMetricConfig.Metric]):
Required. Metrics sources to enable.
"""

class MetricSource(proto.Enum):
r""""""
r"""A source for the collection of Dataproc OSS metrics (see [available
OSS metrics]
(https://cloud.google.com//dataproc/docs/guides/monitoring#available_oss_metrics)).
"""
METRIC_SOURCE_UNSPECIFIED = 0
MONITORING_AGENT_DEFAULTS = 1
HDFS = 2
Expand All @@ -1362,30 +1360,64 @@ class MetricSource(proto.Enum):
SPARK_HISTORY_SERVER = 5
HIVESERVER2 = 6

metric_source = proto.Field(
proto.ENUM,
number=1,
enum=MetricSource,
)
metric_overrides = proto.RepeatedField(
proto.STRING,
number=2,
)


class DataprocMetricConfig(proto.Message):
r"""Specifies a Dataproc metric config
class Metric(proto.Message):
r"""A Dataproc OSS metric.

Attributes:
metric_source (google.cloud.dataproc_v1.types.DataprocMetricConfig.MetricSource):
Required. Default metrics are collected unless
``metricOverrides`` are specified for the metric source (see
[Available OSS metrics]
(https://cloud.google.com/dataproc/docs/guides/monitoring#available_oss_metrics)
for more information).
metric_overrides (Sequence[str]):
Optional. Specify one or more [available OSS metrics]
(https://cloud.google.com/dataproc/docs/guides/monitoring#available_oss_metrics)
to collect for the metric course (for the ``SPARK`` metric
source, any [Spark metric]
(https://spark.apache.org/docs/latest/monitoring.html#metrics)
can be specified).

Provide metrics in the following format:
METRIC_SOURCE:INSTANCE:GROUP:METRIC Use camelcase as
appropriate.

Examples:

::

yarn:ResourceManager:QueueMetrics:AppsCompleted
spark:driver:DAGScheduler:job.allJobs
sparkHistoryServer:JVM:Memory:NonHeapMemoryUsage.committed
hiveserver2:JVM:Memory:NonHeapMemoryUsage.used

Notes:

- Only the specified overridden metrics will be collected
for the metric source. For example, if one or more
``spark:executive`` metrics are listed as metric
overrides, other ``SPARK`` metrics will not be collected.
The collection of the default metrics for other OSS
metric sources is unaffected. For example, if both
``SPARK`` andd ``YARN`` metric sources are enabled, and
overrides are provided for Spark metrics only, all
default YARN metrics will be collected.
"""

Attributes:
metrics (Sequence[google.cloud.dataproc_v1.types.Metric]):
Configuration set of metrics to collect from
the cluster
"""
metric_source = proto.Field(
proto.ENUM,
number=1,
enum="DataprocMetricConfig.MetricSource",
)
metric_overrides = proto.RepeatedField(
proto.STRING,
number=2,
)

metrics = proto.RepeatedField(
proto.MESSAGE,
number=1,
message="Metric",
message=Metric,
)


Expand Down
0