10000 Add entity extraction samples · mmmarklu/python-docs-samples@82c8e04 · GitHub
[go: up one dir, main page]
Skip to content

Commit 82c8e04

Browse files
nnegreynnegrey
committed
Add entity extraction samples
1 parent 76188d7 commit 82c8e04

11 files changed

+500
-1
lines changed

‎automl/cloud-client/deploy_model.py

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2019 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
18+
def deploy_model(project_id, model_id):
19+
"""Deploy a model."""
20+
# [START automl_deploy_model]
21+
from google.cloud import automl
22+
23+
# TODO(developer): Uncomment and saet the following variables
24+
# project_id = 'YOUR_PROJECT_ID'
25+
# model_id = 'YOUR_MODEL_ID'
26+
 8000
27+
client = automl.AutoMlClient()
28+
# Get the full path of the model.
29+
model_full_id = client.model_path(project_id, 'us-central1', model_id)
30+
response = client.deploy_model(model_full_id)
31+
32+
print(u'Model deployment finished. {}'.format(response.result()))
33+
# [END automl_deploy_model]

‎automl/cloud-client/generic_model_management_test.py

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2018 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
import os
18+
19+
import pytest
20+
21+
import list_models
22+
import get_model
23+
import list_model_evaluations
24+
import get_model_evaluation
25+
import list_operation_status
26+
import get_operation_status
27+
import delete_model
28+
29+
PROJECT_ID = os.environ['GCLOUD_PROJECT']
30+
31+
32+
@pytest.mark.slow
33+
def test_list_get_eval_model(capsys):
34+
list_models.list_models(PROJECT_ID)
35+
out, _ = capsys.readouterr()
36+
model_id = out.split('Model id: ')[1].split('\n')[0]
37+
assert 'Model id: ' in out
38+
39+
get_model.get_model(PROJECT_ID, model_id)
40+
out, _ = capsys.readouterr()
41+
assert 'Model id: ' in out
42+
43+
list_model_evaluations.list_model_evaluations(PROJECT_ID, model_id)
44+
out, _ = capsys.readouterr()
45+
model_evaluation_id = \
46+
out.split('{}/modelEvaluations/'.format(model_id))[1].split('\n')[0]
47+
assert 'Model evaluation name: ' in out
48+
49+
get_model_evaluation.get_model_evaluation(
50+
PROJECT_ID, model_id, model_evaluation_id)
51+
out, _ = capsys.readouterr()
52+
assert 'Model evaluation name: ' in out
53+
54+
55+
@pytest.mark.slow
56+
def test_list_get_operation_status(capsys):
57+
list_operation_status.list_operation_status(PROJECT_ID)
58+
out, _ = capsys.readouterr()
59+
operation_id = out.split('Name: ')[1].split('\n')[0]
60+
assert 'Operation details' in out
61+
62+
get_operation_status.get_operation_status(operation_id)
63+
out, _ = capsys.readouterr()
64+
assert 'Operation details' in out
65+
66+
67+
@pytest.mark.slow
68+
def test_delete_model(capsys):
69+
# As model creation can take many hours, instead try to delete a
70+
# nonexistent model and confirm that the model was not found, but other
71+
# elements of the request were valid.
72+
try:
73+
delete_model.delete_model(PROJECT_ID, 'TRL0000000000000000000')
74+
out, _ = capsys.readouterr()
75+
assert 'The model does not exist' in out
76+
except Exception as e:
77+
assert 'The model does not exist' in e.message

‎automl/cloud-client/language_batch_predict.py

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2019 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
18+
def batch_predict(project_id, model_id, input_uri, output_uri):
19+
"""Batch Predict"""
20+
# [START automl_language_batch_predict]
21+
from google.cloud import automl
22+
23+
# TODO(developer): Uncomment and set the following variables
24+
# project_id = 'YOUR_PROJECT_ID'
25+
# model_id = 'YOUR_MODEL_ID'
26+
# input_uri = 'gs://YOUR_BUCKET_ID/path_to_your_input_file.txt'
27+
# output_uri = 'gs://YOUR_BUCKET_ID/path_to_save_results/'
28+
29+
prediction_client = automl.PredictionServiceClient()
30+
31+
# Get the full path of the model.
32+
model_full_id = prediction_client.model_path(
33+
project_id, 'us-central1', model_id)
34+
35+
gcs_source = automl.types.GcsSource(
36+
input_uris=[input_uri])
37+
38+
input_config = automl.types.BatchPredictInputConfig(gcs_source=gcs_source)
39+
gcs_destination = automl.types.GcsDestination(
40+
output_uri_prefix=output_uri)
41+
output_config = automl.types.BatchPredictOutputConfig(
42+
gcs_destination=gcs_destination)
43+
44+
response = prediction_client.batch_predict(
45+
model_full_id, input_config, output_config)
46+
47+
print('Waiting for operation to complete...')
48+
print(u'Batch Prediction results saved to Cloud Storage bucket. {}'.format(
49+
response.result()))
50+
# [END automl_language_batch_predict]

‎automl/cloud-client/language_entity_extraction_create_dataset.py

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2019 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
18+
def create_dataset(project_id, display_name):
19+
"""Create a dataset."""
20+
# [START automl_language_entity_extraction_create_dataset]
21+
from google.cloud import automl
22+
23+
# TODO(developer): Uncomment and set the following variables
24+
# project_id = 'YOUR_PROJECT_ID'
25+
# display_name = 'YOUR_DATASET_NAME'
26+
27+
client = automl.AutoMlClient()
28+
29+
# A resource that represents Google Cloud Platform location.
30+
project_location = client.location_path(project_id, 'us-central1')
31+
metadata = automl.types.TextExtractionDatasetMetadata()
32+
dataset = automl.types.Dataset(
33+
display_name=display_name,
34+
text_extraction_dataset_metadata=metadata)
35+
36+
# Create a dataset with the dataset metadata in the region.
37+
response = client.create_dataset(project_location, dataset)
38+
39+
created_dataset = response.result()
40+
41+
# Display the dataset information
42+
print(u'Dataset name: {}'.format(created_dataset.name))
43+
print(u'Dataset id: {}'.format(created_dataset.name.split('/')[-1]))
44+
# [END automl_language_entity_extraction_create_dataset]
45+
46+
47+
# create_dataset('python-docs-samples-tests', 'operation_known')

‎automl/cloud-client/language_entity_extraction_create_model.py

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2019 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
18+
def create_model(project_id, dataset_id, display_name):
19+
"""Create a model."""
20+
# [START automl_language_entity_extraction_create_model]
21+
from google.cloud import automl
22+
23+
# TODO(developer): Uncomment and set the following variables
24+
# project_id = 'YOUR_PROJECT_ID'
25+
# dataset_id = 'YOUR_DATASET_ID'
26+
# display_name = 'YOUR_MODEL_NAME'
27+
28+
client = automl.AutoMlClient()
29+
30+
# A resource that represents Google Cloud Platform location.
31+
project_location = client.location_path(project_id, 'us-central1')
32+
# Leave model unset to use the default base model provided by Google
33+
metadata = automl.types.TextExtractionModelMetadata()
34+
model = automl.types.Model(
35+
display_name=display_name,
36+
dataset_id=dataset_id,
37+
text_extraction_model_metadata=metadata)
38+
39+
# Create a model with the model metadata in the region.
40+
response = client.create_model(project_location, model)
41+
42+
print(u'Training operation name: {}'.format(response.operation.name))
43+
print('Training started...')
44+
# [END automl_language_entity_extraction_create_model]

‎automl/cloud-client/language_entity_extraction_dataset_management_test.py

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2019 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
import datetime
18+
import os
19+
20+
import pytest
21+
22+
import language_entity_extraction_create_dataset
23+
import import_dataset
24+
import delete_dataset
25+
import list_datasets
26+
import get_dataset
27+
import export_dataset
28+
29+
PROJECT_ID = os.environ['GCLOUD_PROJECT']
30+
BUCKET_ID = '{}-lcm'.format(PROJECT_ID)
31+
DATASET_ID = 'TEN4058147884539838464'
32+
33+
34+
@pytest.mark.slow
35+
def test_create_import_delete_dataset(capsys):
36+
# create dataset
37+
dataset_name = 'test_' + datetime.datetime.now().strftime('%Y%m%d%H%M%S')
38+
language_entity_extraction_create_dataset.create_dataset(PROJECT_ID, dataset_name)
39+
out, _ = capsys.readouterr()
40+
assert 'Dataset id: ' in out
41+
42+
# import data
43+
dataset_id = out.splitlines()[1].split()[2]
44+
data = 'gs://cloud-samples-data/automl/language_entity_extraction/dataset.csv'
45+
import_dataset.import_dataset(PROJECT_ID, dataset_id, data)
46+
out, _ = capsys.readouterr()
47+
assert 'Data imported.' in out
48+
49+
# delete dataset
50+
delete_dataset.delete_dataset(PROJECT_ID, dataset_id)
51+
out, _ = capsys.readouterr()
52+
assert 'Dataset deleted.' in out
53+
54+
55+
def test_list_dataset(capsys):
56+
# list datasets
57+
list_datasets.list_datasets(PROJECT_ID)
58+
out, _ = capsys.readouterr()
59+
assert 'Dataset id: {}'.format(DATASET_ID) in out
60+
61+
62+
def test_get_dataset(capsys):
63+
get_dataset.get_dataset(PROJECT_ID, DATASET_ID)
64+
out, _ = capsys.readouterr()
65+
assert 'Dataset name: ' in out
66+
67+
68+
def test_export_dataset(capsys):
69+
export_dataset.export_dataset(
70+
PROJECT_ID,
71+
DATASET_ID,
72+
'gs://{}/TEST_EXPORT_OUTPUT/'.format(BUCKET_ID))
73+
74+
out, _ = capsys.readouterr()
75+
assert 'Dataset exported' in out
76+
77+
from google.cloud import storage
78+
storage_client = storage.Client()
79+
bucket = storage_client.get_bucket(BUCKET_ID)
80+
if len(list(bucket.list_blobs(prefix='TEST_EXPORT_OUTPUT'))) > 0:
81+
for blob in bucket.list_blobs(prefix='TEST_EXPORT_OUTPUT'):
82+
blob.delete()

‎automl/cloud-client/language_entity_extraction_model_management_test.py

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2018 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
import os
18+
19+
import pytest
20+
21+
import deploy_model
22+
import undeploy_model
23+
24+
PROJECT_ID = os.environ['GCLOUD_PROJECT']
25+
MODEL_ID = 'TEN5112482778553778176'
26+
27+
28+
@pytest.mark.slow
29+
def test_deploy_undeploy_model(capsys):
30+
undeploy_model.undeploy_model(PROJECT_ID, MODEL_ID)
31+
out, _ = capsys.readouterr()
32+
assert 'Model undeployment finished.' in out
33+
34+
deploy_model.deploy_model(PROJECT_ID, MODEL_ID)
35+
out, _ = capsys.readouterr()
36+
assert 'Model deployment finished.' in out

0 commit comments

Comments
 (0)
0