diff --git a/google/cloud/dataproc/__init__.py b/google/cloud/dataproc/__init__.py index f94f9e0c..214250f5 100644 --- a/google/cloud/dataproc/__init__.py +++ b/google/cloud/dataproc/__init__.py @@ -107,7 +107,6 @@ from google.cloud.dataproc_v1.types.clusters import ListClustersResponse from google.cloud.dataproc_v1.types.clusters import ManagedGroupConfig from google.cloud.dataproc_v1.types.clusters import MetastoreConfig -from google.cloud.dataproc_v1.types.clusters import Metric from google.cloud.dataproc_v1.types.clusters import NodeGroupAffinity from google.cloud.dataproc_v1.types.clusters import NodeInitializationAction from google.cloud.dataproc_v1.types.clusters import ReservationAffinity @@ -251,7 +250,6 @@ "ListClustersResponse", "ManagedGroupConfig", "MetastoreConfig", - "Metric", "NodeGroupAffinity", "NodeInitializationAction", "ReservationAffinity", diff --git a/google/cloud/dataproc_v1/__init__.py b/google/cloud/dataproc_v1/__init__.py index d4e6d22f..729f5cbf 100644 --- a/google/cloud/dataproc_v1/__init__.py +++ b/google/cloud/dataproc_v1/__init__.py @@ -71,7 +71,6 @@ from .types.clusters import ListClustersResponse from .types.clusters import ManagedGroupConfig from .types.clusters import MetastoreConfig -from .types.clusters import Metric from .types.clusters import NodeGroupAffinity from .types.clusters import NodeInitializationAction from .types.clusters import ReservationAffinity @@ -231,7 +230,6 @@ "ManagedCluster", "ManagedGroupConfig", "MetastoreConfig", - "Metric", "NodeGroupAffinity", "NodeInitializationAction", "OrderedJob", diff --git a/google/cloud/dataproc_v1/types/__init__.py b/google/cloud/dataproc_v1/types/__init__.py index 4c1baddd..2b3c58ff 100644 --- a/google/cloud/dataproc_v1/types/__init__.py +++ b/google/cloud/dataproc_v1/types/__init__.py @@ -64,7 +64,6 @@ ListClustersResponse, ManagedGroupConfig, MetastoreConfig, - Metric, NodeGroupAffinity, NodeInitializationAction, ReservationAffinity, @@ -193,7 +192,6 @@ "ListClustersResponse", "ManagedGroupConfig", "MetastoreConfig", - "Metric", "NodeGroupAffinity", "NodeInitializationAction", "ReservationAffinity", diff --git a/google/cloud/dataproc_v1/types/clusters.py b/google/cloud/dataproc_v1/types/clusters.py index e55fd3c0..b1b0debf 100644 --- a/google/cloud/dataproc_v1/types/clusters.py +++ b/google/cloud/dataproc_v1/types/clusters.py @@ -47,7 +47,6 @@ "SoftwareConfig", "LifecycleConfig", "MetastoreConfig", - "Metric", "DataprocMetricConfig", "ClusterMetrics", "CreateClusterRequest", @@ -237,7 +236,7 @@ class ClusterConfig(proto.Message): metastore_config (google.cloud.dataproc_v1.types.MetastoreConfig): Optional. Metastore configuration. dataproc_metric_config (google.cloud.dataproc_v1.types.DataprocMetricConfig): - Optional. Dataproc metrics configuration. + Optional. The config for Dataproc metrics. """ config_bucket = proto.Field( @@ -310,7 +309,7 @@ class ClusterConfig(proto.Message): ) dataproc_metric_config = proto.Field( proto.MESSAGE, - number=21, + number=23, message="DataprocMetricConfig", ) @@ -1340,20 +1339,19 @@ class MetastoreConfig(proto.Message): ) -class Metric(proto.Message): - r"""Specifies Dataproc OSS Metric. +class DataprocMetricConfig(proto.Message): + r"""Dataproc metric config. Attributes: - metric_source (google.cloud.dataproc_v1.types.Metric.MetricSource): - Required. Specified source of metric - collection - metric_overrides (Sequence[str]): - Optional. The set of available OSS metrics to - collect from the metric source. + metrics (Sequence[google.cloud.dataproc_v1.types.DataprocMetricConfig.Metric]): + Required. Metrics sources to enable. """ class MetricSource(proto.Enum): - r"""""" + r"""A source for the collection of Dataproc OSS metrics (see [available + OSS metrics] + (https://cloud.google.com//dataproc/docs/guides/monitoring#available_oss_metrics)). + """ METRIC_SOURCE_UNSPECIFIED = 0 MONITORING_AGENT_DEFAULTS = 1 HDFS = 2 @@ -1362,30 +1360,64 @@ class MetricSource(proto.Enum): SPARK_HISTORY_SERVER = 5 HIVESERVER2 = 6 - metric_source = proto.Field( - proto.ENUM, - number=1, - enum=MetricSource, - ) - metric_overrides = proto.RepeatedField( - proto.STRING, - number=2, - ) - - -class DataprocMetricConfig(proto.Message): - r"""Specifies a Dataproc metric config + class Metric(proto.Message): + r"""A Dataproc OSS metric. + + Attributes: + metric_source (google.cloud.dataproc_v1.types.DataprocMetricConfig.MetricSource): + Required. Default metrics are collected unless + ``metricOverrides`` are specified for the metric source (see + [Available OSS metrics] + (https://cloud.google.com/dataproc/docs/guides/monitoring#available_oss_metrics) + for more information). + metric_overrides (Sequence[str]): + Optional. Specify one or more [available OSS metrics] + (https://cloud.google.com/dataproc/docs/guides/monitoring#available_oss_metrics) + to collect for the metric course (for the ``SPARK`` metric + source, any [Spark metric] + (https://spark.apache.org/docs/latest/monitoring.html#metrics) + can be specified). + + Provide metrics in the following format: + METRIC_SOURCE:INSTANCE:GROUP:METRIC Use camelcase as + appropriate. + + Examples: + + :: + + yarn:ResourceManager:QueueMetrics:AppsCompleted + spark:driver:DAGScheduler:job.allJobs + sparkHistoryServer:JVM:Memory:NonHeapMemoryUsage.committed + hiveserver2:JVM:Memory:NonHeapMemoryUsage.used + + Notes: + + - Only the specified overridden metrics will be collected + for the metric source. For example, if one or more + ``spark:executive`` metrics are listed as metric + overrides, other ``SPARK`` metrics will not be collected. + The collection of the default metrics for other OSS + metric sources is unaffected. For example, if both + ``SPARK`` andd ``YARN`` metric sources are enabled, and + overrides are provided for Spark metrics only, all + default YARN metrics will be collected. + """ - Attributes: - metrics (Sequence[google.cloud.dataproc_v1.types.Metric]): - Configuration set of metrics to collect from - the cluster - """ + metric_source = proto.Field( + proto.ENUM, + number=1, + enum="DataprocMetricConfig.MetricSource", + ) + metric_overrides = proto.RepeatedField( + proto.STRING, + number=2, + ) metrics = proto.RepeatedField( proto.MESSAGE, number=1, - message="Metric", + message=Metric, )