8000 Cleanup cluster if error during creation (#7301) · riteshverma/python-docs-samples@8ebe55b · GitHub
[go: up one dir, main page]
Skip to content

Commit 8ebe55b

Browse files
leahecoleleahecole
authored
Cleanup cluster if error during creation (GoogleCloudPlatform#7301)
* Cleanup cluster if error during creation * fix lint run blacken * Use finally instead of catching specific error
1 parent d8d4951 commit 8ebe55b

File tree

4 files changed

+52
-41
lines changed

4 files changed

+52
-41
lines changed

‎data-science-onramp/data-ingestion/ingestion.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,8 @@ def main():
218218

219219
print("Uploading citibike dataset...")
220220
write_to_bigquery(df, CITIBIKE_TABLE_NAME, DATASET_NAME)
221+
222+
221223
# [END datascienceonramp_data_ingestion]
222224

223225

‎data-science-onramp/data-ingestion/ingestion_test.py

Lines changed: 23 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -73,25 +73,31 @@
7373

7474
@pytest.fixture(autouse=True)
7575
def setup_and_teardown_cluster():
76-
# Create cluster using cluster client
77-
cluster_client = dataproc.ClusterControllerClient(
78-
client_options={"api_endpoint": f"{CLUSTER_REGION}-dataproc.googleapis.com:443"}
79-
)
80-
81-
operation = cluster_client.create_cluster(
82-
project_id=PROJECT_ID, region=CLUSTER_REGION, cluster=CLUSTER_CONFIG
83-
)
84-
85-
# Wait for cluster to provision
86-
operation.result()
76+
try:
77+
# Create cluster using cluster client
78+
cluster_client = dataproc.ClusterControllerClient(
79+
client_options={
80+
"api_endpoint": f"{CLUSTER_REGION}-dataproc.googleapis.com:443"
81+
}
82+
)
8783

88-
yield
84+
operation = cluster_client.create_cluster(
85+
project_id=PROJECT_ID, region=CLUSTER_REGION, cluster=CLUSTER_CONFIG
86+
)
8987

90-
# Delete cluster
91-
operation = cluster_client.delete_cluster(
92-
project_id=PROJECT_ID, region=CLUSTER_REGION, cluster_name=DATAPROC_CLUSTER
93-
)
94-
operation.result()
88+
# Wait for cluster to provision
89+
operation.result()
90+
91+
yield
92+
finally:
93+
try:
94+
# Delete cluster
95+
operation = cluster_client.delete_cluster(
96+
project_id=PROJECT_ID, region=CLUSTER_REGION, cluster_name=DATAPROC_CLUSTER
97+
)
98+
operation.result()
99+
except NotFound:
100+
print("Cluster already deleted")
95101

96102

97103
@pytest.fixture(autouse=True)

‎data-science-onramp/data-processing/process.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -168,14 +168,14 @@ def compute_end_udf(duration, start, end):
168168
except Py4JJavaError as e:
169169
raise Exception(f"Error reading {TABLE}") from e
170170

171-
# [END datascienceonramp_sparksession]
171+
# [END datascienceonramp_sparksession]
172172

173-
# [START datascienceonramp_removecolumn]
173+
# [START datascienceonramp_removecolumn]
174174
# remove unused column
175175
df = df.drop("gender")
176-
# [END datascienceonramp_removecolumn]
176+
# [END datascienceonramp_removecolumn]
177177

178-
# [START datascienceonramp_sparksingleudfs]
178+
# [START datascienceonramp_sparksingleudfs]
179179
# Single-parameter udfs
180180
udfs = {
181181
"start_station_name": UserDefinedFunction(station_name_udf, StringType()),

‎data-science-onramp/data-processing/process_test.py

Lines changed: 23 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -105,26 +105,29 @@ def setup_and_teardown_table():
105105

106106
@pytest.fixture(autouse=True)
107107
def setup_and_teardown_cluster():
108-
# Create Dataproc cluster using cluster client
109-
cluster_client = dataproc.ClusterControllerClient(
110-
client_options={"api_endpoint": f"{CLUSTER_REGION}-dataproc.googleapis.com:443"}
111-
)
112-
operation = cluster_client.create_cluster(
113-
project_id=PROJECT_ID, region=CLUSTER_REGION, cluster=CLUSTER_CONFIG
114-
)
115-
116-
# Wait for cluster to provision
117-
operation.result()
118-
119-
yield
120-
121-
# Delete cluster
122-
operation = cluster_client.delete_cluster(
123-
project_id=PROJECT_ID,
124-
region=CLUSTER_REGION,
125-
cluster_name=DATAPROC_CLUSTER
126-
)
127-
operation.result()
108+
try:
109+
# Create Dataproc cluster using cluster client
110+
cluster_client = dataproc.ClusterControllerClient(
111+
client_options={
112+
"api_endpoint": f"{CLUSTER_REGION}-dataproc.googleapis.com:443"
113+
}
114+
)
115+
operation = cluster_client.create_cluster(
116+
project_id=PROJECT_ID, region=CLUSTER_REGION, cluster=CLUSTER_CONFIG
117+
)
118+
119+
# Wait for cluster to provision
120+
operation.result()
121+
yield
122+
finally:
123+
try:
124+
# Delete cluster
125+
operation = cluster_client.delete_cluster(
126+
project_id=PROJECT_ID, region=CLUSTER_REGION, cluster_name=DATAPROC_CLUSTER
127+
)
128+
operation.result()
129+
except NotFound:
130+
print("Cluster already deleted")
128131

129132

130133
@pytest.fixture(autouse=True)

0 commit comments

Comments
 (0)
0