8000 Merge branch 'main' into functions-billing-tests · mortn/python-docs-samples@ad12664 · GitHub
[go: up one dir, main page]
Skip to content

Commit ad12664

Browse files
busunkim96busunkim96
authored
Merge branch 'main' into functions-billing-tests
2 parents 8c5aa74 + 6a0d788 commit ad12664

File tree

6 files changed

+171
-0
lines changed

6 files changed

+171
-0
lines changed

‎dataflow/custom-containers/ubuntu/Dockerfile

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Copyright 2022 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
FROM ubuntu:latest
16+
17+
# Set the entrypoint to Apache Beam SDK worker launcher.
18+
COPY --from=apache/beam_python3.8_sdk:2.36.0 /opt/apache/beam /opt/apache/beam
19+
ENTRYPOINT [ "/opt/apache/beam/boot" ]
20+
21+
# Install Python with pip, dev tools, distutils, and a C++ compiler.
22+
COPY requirements.txt .
23+
RUN apt-get update \
24+
&& apt-get install -y --no-install-recommends \
25+
ca-certificates curl g++ python3.8-dev python3-distutils \
26+
&& rm -rf /var/lib/apt/lists/* \
27+
&& update-alternatives --install /usr/bin/python python /usr/bin/python3.8 10 \
28+
&& curl https://bootstrap.pypa.io/get-pip.py | python \
29+
# Install the requirements.
30+
&& pip install --no-cache-dir -r requirements.txt \
31+
&& pip check
32+
33+
# Copy the pipeline source files.
34 10000 +
COPY main.py ./

‎dataflow/custom-containers/ubuntu/e2e_test.py

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2022 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
import subprocess
18+
19+
try:
20+
# `conftest` cannot be imported when running in `nox`, but we still
21+
# try to import it for the autocomplete when writing the tests.
22+
from conftest import Utils
23+
except ModuleNotFoundError:
24+
Utils = None
25+
import pytest
26+
27+
NAME = "dataflow/custom-containers/minimal"
28+
29+
30+
@pytest.fixture(scope="session")
31+
def bucket_name(utils: Utils) -> str:
32+
yield from utils.storage_bucket(NAME)
33+
34+
35+
@pytest.fixture(scope="session")
36+
def container_image(utils: Utils) -> str:
37+
yield from utils.cloud_build_submit(image_name=NAME)
38+
39+
40+
def test_tensorflow_minimal(
41+
utils: Utils, bucket_name: str, container_image: str
42+
) -> None:
43+
subprocess.check_call(
44+
[
45+
"python",
46+
"main.py",
47+
"--runner=DataflowRunner",
48+
f"--project={utils.project}",
49+
f"--region={utils.region}",
50+
f"--temp_location=gs://{bucket_name}",
51+
f"--sdk_container_image={container_image}",
52+
"--experiment=use_runner_v2",
53+
]
54+
)

‎dataflow/custom-containers/ubuntu/main.py

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2022 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
import logging
18+
import platform
19+
20+
import apache_beam as beam
21+
22+
23+
def run() -> None:
24+
with beam.Pipeline() as pipeline:
25+
(
26+
pipeline
27+
| "Create data" >> beam.Create(["Hello", "World!", platform.platform()])
28+
| "Print" >> beam.Map(logging.info)
29+
)
30+
31+
32+
if __name__ == "__main__":
33+
logging.getLogger().setLevel(logging.INFO)
34+
run()

‎dataflow/custom-containers/ubuntu/noxfile_config.py

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Copyright 2022 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# Default TEST_CONFIG_OVERRIDE for python repos.
16+
17+
# You can copy this file into your directory, then it will be imported from
18+
# the noxfile.py.
19+
20+
# The source of truth:
21+
# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/main/noxfile_config.py
22+
23+
TEST_CONFIG_OVERRIDE = {
24+
# You can opt out from the test for specific Python versions.
25+
# > ℹ️ We're opting out of all Python versions except 3.8.
26+
# > The Python version used is defined by the Dockerfile, so it's redundant
27+
# > to run multiple tests since they would all be running the same Dockerfile.
28+
"ignored_versions": ["2.7", "3.6", "3.7", "3.9", "3.10"],
29+
# Old samples are opted out of enforcing Python type hints
30+
# All new samples should feature them
31+
"enforce_type_hints": True,
32+
# An envvar key for determining the project id to use. Change it
33+
# to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a
34+
# build specific Cloud project. You can also use your own string
35+
# to use your own Cloud project.
36+
"gcloud_project_env": "GOOGLE_CLOUD_PROJECT",
37+
# 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT',
38+
# If you need to use a specific version of pip,
39+
# change pip_version_override to the string representation
40+
# of the version number, for example, "20.2.4"
41+
"pip_version_override": None,
42+
# A dictionary you want to inject into your test. Don't put any
43+
# secrets here. These values will override predefined values.
44+
"envs": {},
45+
}

‎dataflow/custom-containers/ubuntu/requirements-test.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
google-cloud-storage==1.43.0
2+
pytest-xdist==2.5.0
3+
pytest==6.2.4

‎dataflow/custom-containers/ubuntu/requirements.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
apache-beam[gcp]==2.36.0

0 commit comments

Comments
 (0)
0