diff --git a/dataflow/gpu-workers/.dockerignore b/dataflow/gpu-workers/.dockerignore deleted file mode 100644 index 04f5ec66ca6..00000000000 --- a/dataflow/gpu-workers/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -# Ignore files for docker. -.mypy_cache/ -.nox/ -__pycache__/ -env/ -outputs/ diff --git a/dataflow/gpu-workers/.gcloudignore b/dataflow/gpu-workers/.gcloudignore deleted file mode 100644 index cda483971fd..00000000000 --- a/dataflow/gpu-workers/.gcloudignore +++ /dev/null @@ -1,6 +0,0 @@ -# Ignore files for gcloud like Cloud Build. -.mypy_cache/ -.nox/ -__pycache__/ -env/ -outputs/ diff --git a/dataflow/gpu-workers/Dockerfile b/dataflow/gpu-workers/Dockerfile deleted file mode 100644 index 1ddc8fd55e0..00000000000 --- a/dataflow/gpu-workers/Dockerfile +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Make sure the CUDA and cuDNN versions are compatible with your TensorFlow version. -# https://www.tensorflow.org/install/source#gpu -# Check the Nvidia container registry catalog to look at the available Nvidia images: -# https://ngc.nvidia.com/catalog/containers/nvidia:cuda -FROM nvidia/cuda:11.3.1-cudnn8-runtime-ubuntu20.04 - -# The Python version of the Dockerfile MUST match the Python version you use -# to launch the Dataflow job. -ARG python_version=3.8 - -WORKDIR /root - -# Copy the Apache Beam worker files and the requirements.txt file. -COPY --from=apache/beam_python3.8_sdk:2.31.0 /opt/apache/beam /opt/apache/beam -COPY requirements.txt . - -# Update PATH so we find our new Conda and Python installations. -ENV PATH=/opt/python/bin:/opt/conda/bin:$PATH - -RUN apt-get update \ - && apt-get upgrade -y \ - && apt-get install -y wget \ - && rm -rf /var/lib/apt/lists/* \ - # The nvidia image doesn't come with Python pre-installed. - # We use Miniconda to install the Python version of our choice. - && wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ - && sh Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda \ - && rm Miniconda3-latest-Linux-x86_64.sh \ - # Create a new Python environment and install our requirements. - # We don't need to update $PATH since /usr/local is already in $PATH. - && conda create -y -p /opt/python python=$python_version pip \ - && pip install --no-cache-dir -U pip \ - && pip install --no-cache-dir -r requirements.txt \ - && conda clean -y --all --force-pkgs-dirs \ - # Beam workers looks for pip at /usr/local/bin/pip by default. - # This can be omitted in Beam 2.30.0 and later versions. - && ln -s $(which pip) /usr/local/bin/pip - -# Set the entrypoint to Apache Beam SDK worker launcher. -ENTRYPOINT [ "/opt/apache/beam/boot" ] diff --git a/dataflow/gpu-workers/README.md b/dataflow/gpu-workers/README.md deleted file mode 100644 index a71f0da3e95..00000000000 --- a/dataflow/gpu-workers/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Workers with GPUs - -[![Open in Cloud Shell](http://gstatic.com/cloudssh/images/open-btn.svg)](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dataflow/gpu-workers/README.md) - -📝 Tutorial: [Processing Landsat satellite images with GPUs](https://cloud.google.com/dataflow/docs/samples/satellite-images-gpus) diff --git a/dataflow/gpu-workers/cloudbuild.yaml b/dataflow/gpu-workers/cloudbuild.yaml deleted file mode 100644 index dec3d7aabb8..00000000000 --- a/dataflow/gpu-workers/cloudbuild.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# To build the container image: -# PYTHON_VERSION=`python -c 'import platform; print(platform.python_version())'` -# gcloud builds submit --substitutions _PYTHON_VERSION=$PYTHON_VERSION . --timeout 20m - -steps: - # Build the container image with the Python version of our choice. - - name: gcr.io/cloud-builders/docker - args: - [ 'build' - , '--build-arg=python_version=$_PYTHON_VERSION' - , '--tag=gcr.io/$PROJECT_ID/$_IMAGE' - , '.' - ] - - # Push the image to Container Registry. - - name: gcr.io/cloud-builders/docker - args: [ 'push', 'gcr.io/$PROJECT_ID/$_IMAGE' ] - -substitutions: - _PYTHON_VERSION: '3.8' - _IMAGE: samples/dataflow/tensorflow-gpu:latest diff --git a/dataflow/gpu-workers/e2e_test.py b/dataflow/gpu-workers/e2e_test.py deleted file mode 100644 index f3f105d7b6a..00000000000 --- a/dataflow/gpu-workers/e2e_test.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import platform -import subprocess -import uuid - -from google.cloud import storage -import pytest - -SUFFIX = uuid.uuid4().hex[0:6] -PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] -BUCKET_NAME = f"dataflow-gpu-test-{SUFFIX}" -IMAGE_NAME = f"dataflow/gpu-workers/test-{SUFFIX}:latest" -REGION = "us-central1" -ZONE = "us-central1-f" - - -@pytest.fixture(scope="session") -def bucket_name() -> str: - storage_client = storage.Client() - bucket = storage_client.create_bucket(BUCKET_NAME) - - yield BUCKET_NAME - - bucket.delete(force=True) - - -@pytest.fixture(scope="session") -def configure_docker() -> None: - subprocess.run( - [ - "gcloud", - "auth", - "configure-docker", - ] - ) - - -@pytest.fixture(scope="session") -def image_name(configure_docker: None) -> str: - # See the `cloudbuild.yaml` for the configuration for this build. - substitutions = { - "_PYTHON_VERSION": platform.python_version(), - "_IMAGE": IMAGE_NAME, - } - print(f"-- Cloud build substitutions: {substitutions}") - subprocess.run( - [ - "gcloud", - "builds", - "submit", - f"--project={PROJECT}", - f"--substitutions={','.join([k + '=' + v for k, v in substitutions.items()])}", - "--timeout=30m", - "--quiet", - ], - check=True, - ) - - yield f"gcr.io/{PROJECT}/{IMAGE_NAME}" - - # Delete the image when we're done. - subprocess.run( - [ - "gcloud", - "container", - "images", - "delete", - f"gcr.io/{PROJECT}/{IMAGE_NAME}", - f"--project={PROJECT}", - "--quiet", - ], - check=True, - ) - - -def test_end_to_end(bucket_name: str, image_name: str) -> None: - # Run the Beam pipeline in Dataflow making sure GPUs are used. - gpu_type = "nvidia-tesla-t4" - subprocess.run( - [ - "python", - "landsat_view.py", - f"--output-path-prefix=gs://{bucket_name}/outputs/", - "--runner=DataflowRunner", - f"--job_name=gpu-workers-{SUFFIX}", - f"--project={PROJECT}", - f"--region={REGION}", - f"--temp_location=gs://{bucket_name}/temp", - "--worker_machine_type=custom-1-13312-ext", - "--disk_size_gb=300", - f"--worker_harness_container_image={image_name}", - f"--worker_zone={ZONE}", - f"--experiments=worker_accelerator=type={gpu_type},count=1,install-nvidia-driver", - "--experiments=use_runner_v2", - ], - check=True, - ) - - # Check that output files were created and are not empty. - storage_client = storage.Client() - output_files = list(storage_client.list_blobs(bucket_name, prefix="outputs/")) - assert len(output_files) > 0, "No output files found" - for output_file in output_files: - assert output_file.size > 0, f"Output file is empty: {output_file.name}" diff --git a/dataflow/gpu-workers/landsat_view.py b/dataflow/gpu-workers/landsat_view.py deleted file mode 100644 index 9e61016eabf..00000000000 --- a/dataflow/gpu-workers/landsat_view.py +++ /dev/null @@ -1,368 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This Apache Beam pipeline processes Landsat 8 satellite images and renders -them as JPEG files. - -A Landsat 8 image consists of 11 bands. Each band contains the data for a -specific range of the electromagnetic spectrum. - -A JPEG image consists of three channels: Red, Green, and Blue. For Landsat 8 -images, these correspond to Band 4 (red), Band 3 (green), and Band 2 (blue). - -These bands contain the raw pixel data directly from the satellite sensors. The -values in each band can go from 0 to unbounded positive values. For a JPEG image -we need to clamp them into integers between 0 and 255 for each channel. - -For this, we supply visualization parameters, commonly called `vis_params`. -These visualization parameters include: - -- The bands for the RGB cannels, typically [B4, B3, B2] for Landsat 8. -- The minimum value in each band, typically 0 for Landsat 8. -- The maximum value in each band, this varies depending on the light exposure. -- A gamma value for gamma correction. - -The Landsat data is read from the Landsat public dataset in Cloud Storage. -For more information on the Landsat dataset: - https://cloud.google.com/storage/docs/public-datasets/landsat - -The overall workflow of the pipeline is the following: - -- Parse one or more Landsat scene IDs from user-provided flags.. -- Get the Cloud Storage paths of all the RGB bands. -- Load the pixel values for each band from Cloud Storage. -- Preprocess pixels: clamp values and apply gamma correction. -- Create a JPEG image and save it to Cloud Storage. -""" - -import argparse -import logging -import os -import re -from typing import Any, Dict, List, Tuple - -import apache_beam as beam -from apache_beam.options.pipeline_options import PipelineOptions -from apache_beam.typehints.typehints import Optional -import numpy as np -from PIL import Image -import rasterio -import tensorflow as tf - -DEFAULT_RGB_BAND_NAMES = ["B4", "B3", "B2"] -DEFAULT_MIN_BAND_VALUE = 0.0 -DEFAULT_MAX_BAND_VALUE = 12000.0 -DEFAULT_GAMMA = 0.5 - -DEFAULT_SCENES = [ - "LC08_L1TP_001067_20200727_20200807_01_T1", # Brazil-Bolivia boundary - "LC08_L1TP_019024_20190621_20190704_01_T1", # Nottaway river delta, Quebec - "LC08_L1TP_019046_20191214_20191226_01_T1", # Yucatan peninsula - "LC08_L1TP_037035_20191212_20191212_01_T1", # Grand canyon, Arizona - "LC08_L1TP_045031_20200715_20200722_01_T1", # Mount Shasta, California - "LC08_L1TP_064011_20200618_20200625_01_T1", # Mackenzie river delta, Canada - "LC08_L1TP_073087_20200516_20200527_01_T1", # Mt. Taranaki, New Zealand - "LC08_L1TP_083074_20180805_20180814_01_T1", # Nouvelle-Calédonie - "LC08_L1TP_098063_20200703_20200708_01_T1", # Manam volcano, Papua New Guinea - "LC08_L1TP_109078_20200411_20200422_01_T1", # Lake Carnegie, West Australia - "LC08_L1TP_110036_20191009_20191018_01_T1", # Osaka 大阪市, Japan - "LC08_L1TP_115078_20200608_20200625_01_T1", # Sediment deposits, West Australia - "LC08_L1TP_119038_20191109_20191115_01_T1", # Lake Tai 太湖, China - "LC08_L1TP_135040_20190314_20190325_01_T1", # Arunachal Pradesh, India - "LC08_L1TP_137045_20200211_20200225_01_T1", # Ganges river delta, India - "LC08_L1TP_166075_20180608_20180615_01_T1", # Bazaruto island, Mozambique - "LC08_L1TP_169034_20200720_20200807_01_T1", # Lake Urmia دریاچه ارومیه, Iran - "LC08_L1TP_170059_20200101_20200113_01_T1", # Mount Elgon, Uganda - "LC08_L1TP_175079_20200511_20200526_01_T1", # Sand dunes, South Africa - "LC08_L1TP_178069_20200804_20200821_01_T1", # Angola - "LC08_L1TP_178078_20200804_20200821_01_T1", # Sand dunes, Namibia - "LC08_L1TP_191020_20200815_20200822_01_T1", # Phytoplankton at Gotland, Sweden - "LC08_L1TP_195028_20200116_20200127_01_T1", # Swiss Alps - "LC08_L1TP_203045_20200108_20200114_01_T1", # Eye of the Sahara, Mauritania - "LC08_L1TP_231094_20190906_20190917_01_T1", # Patagonia, South America -] - -SCENE_RE = re.compile( - r"(?PL[COTEM]0[78])_" - r"(?PL1TP|L1GT|L1GS)_" - r"(?P\d\d\d)" - r"(?P\d\d\d)_" - r"(?P\d\d\d\d)" - r"(?P\d\d)" - r"(?P\d\d)_" - r"(?P\d\d\d\d)" - r"(?P\d\d)" - r"(?P\d\d)_" - r"(?P\d\d)_" - r"(?PRT|T1|T2)" -) - - -def check_gpus(element: Any, gpus_optional: bool) -> Any: - """Makes sure TensorFlow detects GPUs, otherwise raise a RuntimeError. - - Note that this function must be run within a PTransform like beam.Map so - we are sure it's run by the workers, and not the launcher process. - - Args: - element: An element - gpus_optional: If True, the pipeline won't crash if GPUs are not found. - - Returns: - The same element it received as is. - - Raises: - RuntimeError: If no GPUs were found by TensorFlow. - """ - # Make sure we have a GPU available. - gpu_devices = tf.config.list_physical_devices("GPU") - logging.info(f"GPU devices: {gpu_devices}") - if len(gpu_devices) == 0: - if gpus_optional: - logging.warning("No GPUs found, defaulting to CPU.") - else: - raise RuntimeError("No GPUs found.") - return element - - -def get_band_paths( - scene: str, band_names: List[str], unused_side_input: Any -) -> Tuple[str, List[str]]: - """Gets the Cloud Storage paths for each band in a Landsat scene. - - Args: - scene: Landsat 8 scene ID. - band_names: List of the band names corresponding to [Red, Green, Blue] channels. - unused_side_input: Used to wait for the GPU check, can be safely ignored. - - Returns: - A (scene, band_paths) pair. - - Raises: - ValueError: If the scene or a band does not exist. - """ - # Extract the metadata from the scene ID using a regular expression. - m = SCENE_RE.match(scene) - if not m: - raise ValueError(f"invalid scene ID: {scene}") - - g = m.groupdict() - scene_dir = f"gs://gcp-public-data-landsat/{g['sensor']}/{g['collection']}/{g['wrs_path']}/{g['wrs_row']}/{scene}" - - band_paths = [f"{scene_dir}/{scene}_{band_name}.TIF" for band_name in band_names] - - for band_path in band_paths: - if not tf.io.gfile.exists(band_path): - raise ValueError(f"failed to load: {band_path}") - - return scene, band_paths - - -def load_values(scene: str, band_paths: List[str]) -> Tuple[str, np.ndarray]: - """Loads a scene's bands data as a numpy array. - - Args: - scene: Landsat 8 scene ID. - band_paths: A list of the [Red, Green, Blue] band paths. - - Returns: - A (scene, values) pair. - - The values are stored in a three-dimensional float32 array with shape: - (band, width, height) - """ - - def read_band(band_path: str) -> np.array: - # Use rasterio to read the GeoTIFF values from the band files. - with tf.io.gfile.GFile(band_path, "rb") as f, rasterio.open(f) as data: - return data.read(1) - - logging.info(f"{scene}: load_values({band_paths})") - values = [read_band(band_path) for band_path in band_paths] - return scene, np.array(values, np.float32) - - -def preprocess_pixels( - scene: str, - values: np.ndarray, - min_value: float = 0.0, - max_value: float = 1.0, - gamma: float = 1.0, -) -> Tuple[str, tf.Tensor]: - """Prepares the band data into a pixel-ready format for an RGB image. - - The input band values come in the shape (band, width, height) with - unbounded positive numbers depending on the sensor's exposure. - The values are reshaped into (width, height, band), the values are clamped - to integers between 0 and 255, and a gamma correction value is applied. - - Args: - scene: Landsat 8 scene ID. - values: Band values in the shape (band, width, height). - min_value: Minimum band value. - max_value: Maximum band value. - gamma: Gamma correction value. - - Returns: - A (scene, pixels) pair. The pixels are Image-ready values. - """ - logging.info( - f"{scene}: preprocess_pixels({values.shape}:{values.dtype}, min={min_value}, max={max_value}, gamma={gamma})" - ) - - # Reshape (band, width, height) into (width, height, band). - pixels = tf.transpose(values, (1, 2, 0)) - - # Rescale to values from 0.0 to 1.0 and clamp them into that range. - pixels -= min_value - pixels /= max_value - pixels = tf.clip_by_value(pixels, 0.0, 1.0) - - # Apply gamma correction. - pixels **= 1.0 / gamma - - # Return the pixel values as int8 in the range from 0 to 255, - # which is what PIL.Image expects. - return scene, tf.cast(pixels * 255.0, dtype=tf.uint8) - - -def save_to_gcs( - scene: str, image: Image.Image, output_path_prefix: str, format: str = "JPEG" -) -> None: - """Saves a PIL.Image as a JPEG file in the desired path. - - Args: - scene: Landsat 8 scene ID. - image: A PIL.Image object. - output_path_prefix: Path prefix to save the output files. - format: Image format to save files. - """ - filename = os.path.join(output_path_prefix, scene + "." + format.lower()) - with tf.io.gfile.GFile(filename, "w") as f: - image.save(f, format) - - -def run( - scenes: List[str], - output_path_prefix: str, - vis_params: Dict[str, Any], - gpus_optional: bool, - beam_args: Optional[List[str]] = None, -) -> None: - """Load multiple Landsat scenes and render them as JPEG files. - - Args: - scenes: List of Landsat 8 scene IDs. - output_path_prefix: Path prefix to save the output files. - vis_params: Visualization parameters including {rgb_bands, min, max, gamma}. - gpus_optional: If True, the pipeline won't crash if GPUs are not found. - beam_args: Optional list of arguments for Beam pipeline options. - """ - rgb_band_names = vis_params["rgb_band_names"] - min_value = vis_params["min"] - max_value = vis_params["max"] - gamma = vis_params["gamma"] - - options = PipelineOptions(beam_args, save_main_session=True) - with beam.Pipeline(options=options) as pipeline: - # Optionally, validate that the workers are using GPUs. - gpu_check = ( - pipeline - | beam.Create([None]) - | "Check GPU availability" >> beam.Map(check_gpus, gpus_optional) - ) - - # Convert Landsat 8 scenes into images. - # ℹ️ We pass `gpu_check` as an unused side input to force that step in - # the pipeline to wait for the check before continuing. - ( - pipeline - | "Create scene IDs" >> beam.Create(scenes) - | "Get RGB band paths" - >> beam.Map( - get_band_paths, - rgb_band_names, - unused_side_input=beam.pvalue.AsSingleton(gpu_check), - ) - | "Load RGB band values" >> beam.MapTuple(load_values) - | "Preprocess pixels" - >> beam.MapTuple(preprocess_pixels, min_value, max_value, gamma) - | "Convert to image" - >> beam.MapTuple( - lambda scene, rgb_pixels: ( - scene, - Image.fromarray(rgb_pixels.numpy(), mode="RGB"), - ) - ) - | "Save to Cloud Storage" >> beam.MapTuple(save_to_gcs, output_path_prefix) - ) - - -if __name__ == "__main__": - logging.getLogger().setLevel(logging.INFO) - - parser = argparse.ArgumentParser() - parser.add_argument( - "--output-path-prefix", - required=True, - help="Path prefix for output image files. " - "This can be a Google Cloud Storage path.", - ) - parser.add_argument( - "--scene", - dest="scenes", - action="append", - help="One or more Landsat scene IDs to process, for example " - "LC08_L1TP_109078_20200411_20200422_01_T1. " - "They must be in the format: " - "https://www.usgs.gov/faqs/what-naming-convention-landsat-collections-level-1-scenes", - ) - parser.add_argument( - "--rgb-band-names", - nargs=3, - default=DEFAULT_RGB_BAND_NAMES, - help="List of three band names to be mapped to the RGB channels.", - ) - parser.add_argument( - "--min", - type=float, - default=DEFAULT_MIN_BAND_VALUE, - help="Minimum value of the band value range.", - ) - parser.add_argument( - "--max", - type=float, - default=DEFAULT_MAX_BAND_VALUE, - help="Maximum value of the band value range.", - ) - parser.add_argument( - "--gamma", type=float, default=DEFAULT_GAMMA, help="Gamma correction factor." - ) - parser.add_argument( - "--gpus-optional", - action="store_true", - help="If set, the pipeline won't crash if GPUs are not found.", - ) - args, beam_args = parser.parse_known_args() - - scenes = args.scenes or DEFAULT_SCENES - vis_params = { - "rgb_band_names": args.rgb_band_names, - "min": args.min, - "max": args.max, - "gamma": args.gamma, - } - run(scenes, args.output_path_prefix, vis_params, args.gpus_optional, beam_args) diff --git a/dataflow/gpu-workers/noxfile_config.py b/dataflow/gpu-workers/noxfile_config.py deleted file mode 100644 index 74d736256c6..00000000000 --- a/dataflow/gpu-workers/noxfile_config.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default TEST_CONFIG_OVERRIDE for python repos. - -# You can copy this file into your directory, then it will be imported from -# the noxfile.py. - -# The source of truth: -# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py - -TEST_CONFIG_OVERRIDE = { - # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7", "3.9"], - # Old samples are opted out of enforcing Python type hints - # All new samples should feature them - "enforce_type_hints": True, - # An envvar key for determining the project id to use. Change it - # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a - # build specific Cloud project. You can also use your own string - # to use your own Cloud project. - "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", - # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', - # A dictionary you want to inject into your test. Don't put any - # secrets here. These values will override predefined values. - "envs": {}, -} diff --git a/dataflow/gpu-workers/requirements-test.txt b/dataflow/gpu-workers/requirements-test.txt deleted file mode 100644 index 9782f5d8d54..00000000000 --- a/dataflow/gpu-workers/requirements-test.txt +++ /dev/null @@ -1,2 +0,0 @@ -google-cloud-storage==1.38.0 -pytest==6.2.4 diff --git a/dataflow/gpu-workers/requirements.txt b/dataflow/gpu-workers/requirements.txt deleted file mode 100644 index 765c53897c2..00000000000 --- a/dataflow/gpu-workers/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -Pillow==8.3.0 -apache-beam[gcp]==2.30.0 -rasterio==1.2.6 -tensorflow==2.5.1