|
| 1 | +# Copyright 2025 Google LLC |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | +# limitations under the License. |
| 14 | + |
| 15 | +from __future__ import annotations |
| 16 | + |
| 17 | +import logging |
| 18 | + |
| 19 | +from google.cloud import exceptions as cloud_exceptions |
| 20 | +from google.cloud import storage |
| 21 | +from typing_extensions import override |
| 22 | + |
| 23 | +from ..errors.not_found_error import NotFoundError |
| 24 | +from ._eval_set_results_manager_utils import create_eval_set_result |
| 25 | +from .eval_result import EvalCaseResult |
| 26 | +from .eval_result import EvalSetResult |
| 27 | +from .eval_set_results_manager import EvalSetResultsManager |
| 28 | + |
| 29 | +logger = logging.getLogger("google_adk." + __name__) |
| 30 | + |
| 31 | +_EVAL_HISTORY_DIR = "evals/eval_history" |
| 32 | +_EVAL_SET_RESULT_FILE_EXTENSION = ".evalset_result.json" |
| 33 | + |
| 34 | + |
| 35 | +class GcsEvalSetResultsManager(EvalSetResultsManager): |
| 36 | + """An EvalSetResultsManager that stores eval results in a GCS bucket.""" |
| 37 | + |
| 38 | + def __init__(self, bucket_name: str, **kwargs): |
| 39 | + """Initializes the GcsEvalSetsManager. |
| 40 | +
|
| 41 | + Args: |
| 42 | + bucket_name: The name of the bucket to use. |
| 43 | + **kwargs: Keyword arguments to pass to the Google Cloud Storage client. |
| 44 | + """ |
| 45 | + self.bucket_name = bucket_name |
| 46 | + self.storage_client = storage.Client(**kwargs) |
| 47 | + self.bucket = self.storage_client.bucket(self.bucket_name) |
| 48 | + # Check if the bucket exists. |
| 49 | + if not self.bucket.exists(): |
| 50 | + raise ValueError( |
| 51 | + f"Bucket `{self.bucket_name}` does not exist. Please create it before" |
| 52 | + " using the GcsEvalSetsManager." |
| 53 | + ) |
| 54 | + |
| 55 | + def _get_eval_history_dir(self, app_name: str) -> str: |
| 56 | + return f"{app_name}/{_EVAL_HISTORY_DIR}" |
| 57 | + |
| 58 | + def _get_eval_set_result_blob_name( |
| 59 | + self, app_name: str, eval_set_result_id: str |
| 60 | + ) -> str: |
| 61 | + eval_history_dir = self._get_eval_history_dir(app_name) |
| 62 | + return f"{eval_history_dir}/{eval_set_result_id}{_EVAL_SET_RESULT_FILE_EXTENSION}" |
| 63 | + |
| 64 | + def _write_eval_set_result( |
| 65 | + self, blob_name: str, eval_set_result: EvalSetResult |
| 66 | + ): |
| 67 | + """Writes an EvalSetResult to GCS.""" |
| 68 | + blob = self.bucket.blob(blob_name) |
| 69 | + blob.upload_from_string( |
| 70 | + eval_set_result.model_dump_json(indent=2), |
| 71 | + content_type="application/json", |
| 72 | + ) |
| 73 | + |
| 74 | + @override |
| 75 | + def save_eval_set_result( |
| 76 | + self, |
| 77 | + app_name: str, |
| 78 | + eval_set_id: str, |
| 79 | + eval_case_results: list[EvalCaseResult], |
| 80 | + ) -> None: |
| 81 | + """Creates and saves a new EvalSetResult given eval_case_results.""" |
| 82 | + eval_set_result = create_eval_set_result( |
| 83 | + app_name, eval_set_id, eval_case_results |
| 84 | + ) |
| 85 | + |
| 86 | + eval_set_result_blob_name = self._get_eval_set_result_blob_name( |
| 87 | + app_name, eval_set_result.eval_set_result_id |
| 88 | + ) |
| 89 | + logger.info("Writing eval result to blob: %s", eval_set_result_blob_name) |
| 90 | + self._write_eval_set_result(eval_set_result_blob_name, eval_set_result) |
| 91 | + |
| 92 | + @override |
| 93 | + def get_eval_set_result( |
| 94 | + self, app_name: str, eval_set_result_id: str |
| 95 | + ) -> EvalSetResult: |
| 96 | + """Returns an EvalSetResult from app_name and eval_set_result_id.""" |
| 97 | + eval_set_result_blob_name = self._get_eval_set_result_blob_name( |
| 98 | + app_name, eval_set_result_id |
| 99 | + ) |
| 100 | + blob = self.bucket.blob(eval_set_result_blob_name) |
| 101 | + if not blob.exists(): |
| 102 | + raise NotFoundError(f"Eval set result `{eval_set_result_id}` not found.") |
| 103 | + eval_set_result_data = blob.download_as_text() |
| 104 | + return EvalSetResult.model_validate_json(eval_set_result_data) |
| 105 | + |
| 106 | + @override |
| 107 | + def list_eval_set_results(self, app_name: str) -> list[str]: |
| 108 | + """Returns the eval result ids that belong to the given app_name.""" |
| 109 | + eval_history_dir = self._get_eval_history_dir(app_name) |
| 110 | + eval_set_results = [] |
| 111 | + try: |
| 112 | + for blob in self.bucket.list_blobs(prefix=eval_history_dir): |
| 113 | + eval_set_result_id = blob.name.split("/")[-1].removesuffix( |
| 114 | + _EVAL_SET_RESULT_FILE_EXTENSION |
| 115 | + ) |
| 116 | + eval_set_results.append(eval_set_result_id) |
| 117 | + return sorted(eval_set_results) |
| 118 | + except cloud_exceptions.NotFound as e: |
| 119 | + raise ValueError( |
| 120 | + f"App `{app_name}` not found in GCS bucket `{self.bucket_name}`." |
| 121 | + ) from e |
0 commit comments