diff --git a/libraries/botbuilder-ai/botbuilder/ai/qna/__init__.py b/libraries/botbuilder-ai/botbuilder/ai/qna/__init__.py index 5893edfc8..dc3f3ccba 100644 --- a/libraries/botbuilder-ai/botbuilder/ai/qna/__init__.py +++ b/libraries/botbuilder-ai/botbuilder/ai/qna/__init__.py @@ -1,18 +1,35 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from .metadata import Metadata -from .query_result import QueryResult from .qnamaker import QnAMaker from .qnamaker_endpoint import QnAMakerEndpoint from .qnamaker_options import QnAMakerOptions from .qnamaker_telemetry_client import QnAMakerTelemetryClient -from .qna_telemetry_constants import QnATelemetryConstants -from .qnamaker_trace_info import QnAMakerTraceInfo +from .utils import ( + ActiveLearningUtils, + GenerateAnswerUtils, + HttpRequestUtils, + QnATelemetryConstants, +) + +from .models import ( + FeedbackRecord, + FeedbackRecords, + Metadata, + QnAMakerTraceInfo, + QueryResult, + QueryResults, +) __all__ = [ + "ActiveLearningUtils", + "FeedbackRecord", + "FeedbackRecords", + "GenerateAnswerUtils", + "HttpRequestUtils", "Metadata", "QueryResult", + "QueryResults", "QnAMaker", "QnAMakerEndpoint", "QnAMakerOptions", diff --git a/libraries/botbuilder-ai/botbuilder/ai/qna/metadata.py b/libraries/botbuilder-ai/botbuilder/ai/qna/metadata.py deleted file mode 100644 index aa68e5811..000000000 --- a/libraries/botbuilder-ai/botbuilder/ai/qna/metadata.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - - -class Metadata: - def __init__(self, name, value): - self.name = name - self.value = value diff --git a/libraries/botbuilder-ai/botbuilder/ai/qna/models/__init__.py b/libraries/botbuilder-ai/botbuilder/ai/qna/models/__init__.py new file mode 100644 index 000000000..baaa22063 --- /dev/null +++ b/libraries/botbuilder-ai/botbuilder/ai/qna/models/__init__.py @@ -0,0 +1,26 @@ +# coding=utf-8 +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +from .feedback_record import FeedbackRecord +from .feedback_records import FeedbackRecords +from .generate_answer_request_body import GenerateAnswerRequestBody +from .metadata import Metadata +from .qnamaker_trace_info import QnAMakerTraceInfo +from .query_result import QueryResult +from .query_results import QueryResults +from .train_request_body import TrainRequestBody + +__all__ = [ + "FeedbackRecord", + "FeedbackRecords", + "GenerateAnswerRequestBody", + "Metadata", + "QnAMakerTraceInfo", + "QueryResult", + "QueryResults", + "TrainRequestBody", +] diff --git a/libraries/botbuilder-ai/botbuilder/ai/qna/models/feedback_record.py b/libraries/botbuilder-ai/botbuilder/ai/qna/models/feedback_record.py new file mode 100644 index 000000000..74a78d5d0 --- /dev/null +++ b/libraries/botbuilder-ai/botbuilder/ai/qna/models/feedback_record.py @@ -0,0 +1,32 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from msrest.serialization import Model + + +class FeedbackRecord(Model): + """ Active learning feedback record. """ + + _attribute_map = { + "user_id": {"key": "userId", "type": "str"}, + "user_question": {"key": "userQuestion", "type": "str"}, + "qna_id": {"key": "qnaId", "type": "int"}, + } + + def __init__(self, user_id: str, user_question: str, qna_id: int, **kwargs): + """ + Parameters: + ----------- + + user_id: ID of the user. + + user_question: User question. + + qna_id: QnA ID. + """ + + super().__init__(**kwargs) + + self.user_id = user_id + self.user_question = user_question + self.qna_id = qna_id diff --git a/libraries/botbuilder-ai/botbuilder/ai/qna/models/feedback_records.py b/libraries/botbuilder-ai/botbuilder/ai/qna/models/feedback_records.py new file mode 100644 index 000000000..62f3983c4 --- /dev/null +++ b/libraries/botbuilder-ai/botbuilder/ai/qna/models/feedback_records.py @@ -0,0 +1,26 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import List + +from msrest.serialization import Model + +from .feedback_record import FeedbackRecord + + +class FeedbackRecords(Model): + """ Active learning feedback records. """ + + _attribute_map = {"records": {"key": "records", "type": "[FeedbackRecord]"}} + + def __init__(self, records: List[FeedbackRecord], **kwargs): + """ + Parameter(s): + ------------- + + records: List of feedback records. + """ + + super().__init__(**kwargs) + + self.records = records diff --git a/libraries/botbuilder-ai/botbuilder/ai/qna/models/generate_answer_request_body.py b/libraries/botbuilder-ai/botbuilder/ai/qna/models/generate_answer_request_body.py new file mode 100644 index 000000000..6dba9a124 --- /dev/null +++ b/libraries/botbuilder-ai/botbuilder/ai/qna/models/generate_answer_request_body.py @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import List + +from msrest.serialization import Model + +from .metadata import Metadata + + +class GenerateAnswerRequestBody(Model): + """ Question used as the payload body for QnA Maker's Generate Answer API. """ + + _attribute_map = { + "question": {"key": "question", "type": "str"}, + "top": {"key": "top", "type": "int"}, + "score_threshold": {"key": "scoreThreshold", "type": "float"}, + "strict_filters": {"key": "strictFilters", "type": "[Metadata]"}, + } + + def __init__( + self, + question: str, + top: int, + score_threshold: float, + strict_filters: List[Metadata], + **kwargs + ): + """ + Parameters: + ----------- + + question: The user question to query against the knowledge base. + + top: Max number of answers to be returned for the question. + + score_threshold: Threshold for answers returned based on score. + + strict_filters: Find only answers that contain these metadata. + """ + + super().__init__(**kwargs) + + self.question = question + self.top = top + self.score_threshold = score_threshold + self.strict_filters = strict_filters diff --git a/libraries/botbuilder-ai/botbuilder/ai/qna/models/metadata.py b/libraries/botbuilder-ai/botbuilder/ai/qna/models/metadata.py new file mode 100644 index 000000000..af0f1f00b --- /dev/null +++ b/libraries/botbuilder-ai/botbuilder/ai/qna/models/metadata.py @@ -0,0 +1,28 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from msrest.serialization import Model + + +class Metadata(Model): + """ Metadata associated with the answer. """ + + _attribute_map = { + "name": {"key": "name", "type": "str"}, + "value": {"key": "value", "type": "str"}, + } + + def __init__(self, name: str, value: str, **kwargs): + """ + Parameters: + ----------- + + name: Metadata name. Max length: 100. + + value: Metadata value. Max length: 100. + """ + + super().__init__(**kwargs) + + self.name = name + self.value = value diff --git a/libraries/botbuilder-ai/botbuilder/ai/qna/models/qnamaker_trace_info.py b/libraries/botbuilder-ai/botbuilder/ai/qna/models/qnamaker_trace_info.py new file mode 100644 index 000000000..22119687d --- /dev/null +++ b/libraries/botbuilder-ai/botbuilder/ai/qna/models/qnamaker_trace_info.py @@ -0,0 +1,44 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import List + +from botbuilder.schema import Activity +from .metadata import Metadata +from .query_result import QueryResult + + +class QnAMakerTraceInfo: + """ Represents all the trice info that we collect from the QnAMaker Middleware. """ + + def __init__( + self, + message: Activity, + query_results: List[QueryResult], + knowledge_base_id: str, + score_threshold: float, + top: int, + strict_filters: List[Metadata], + ): + """ + Parameters: + ----------- + + message: Message which instigated the query to QnA Maker. + + query_results: Results that QnA Maker returned. + + knowledge_base_id: ID of the knowledge base that is being queried. + + score_threshold: The minimum score threshold, used to filter returned results. + + top: Number of ranked results that are asked to be returned. + + strict_filters: Filters used on query. + """ + self.message = message + self.query_results = query_results + self.knowledge_base_id = knowledge_base_id + self.score_threshold = score_threshold + self.top = top + self.strict_filters = strict_filters diff --git a/libraries/botbuilder-ai/botbuilder/ai/qna/models/query_result.py b/libraries/botbuilder-ai/botbuilder/ai/qna/models/query_result.py new file mode 100644 index 000000000..387eb8796 --- /dev/null +++ b/libraries/botbuilder-ai/botbuilder/ai/qna/models/query_result.py @@ -0,0 +1,42 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import List + +from .metadata import Metadata + + +class QueryResult: + """ Represents an individual result from a knowledge base query. """ + + def __init__( + self, + questions: List[str], + answer: str, + score: float, + metadata: object = None, + source: str = None, + id: int = None, # pylint: disable=invalid-name + ): + """ + Parameters: + ----------- + + questions: The list of questions indexed in the QnA Service for the given answer (if any). + + answer: Answer from the knowledge base. + + score: Confidence on a scale from 0.0 to 1.0 that the answer matches the user's intent. + + metadata: Metadata associated with the answer (if any). + + source: The source from which the QnA was extracted (if any). + + id: The index of the answer in the knowledge base. V3 uses 'qnaId', V4 uses 'id' (if any). + """ + self.questions = questions + self.answer = answer + self.score = score + self.metadata = list(map(lambda meta: Metadata(**meta), metadata)) + self.source = source + self.id = id # pylint: disable=invalid-name diff --git a/libraries/botbuilder-ai/botbuilder/ai/qna/models/query_results.py b/libraries/botbuilder-ai/botbuilder/ai/qna/models/query_results.py new file mode 100644 index 000000000..3983ca351 --- /dev/null +++ b/libraries/botbuilder-ai/botbuilder/ai/qna/models/query_results.py @@ -0,0 +1,19 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import List + +from .query_result import QueryResult + + +class QueryResults: + """ Contains answers for a user query. """ + + def __init__(self, answers: List[QueryResult]): + """ + Parameters: + ----------- + + answers: The answers for a user query. + """ + self.answers = answers diff --git a/libraries/botbuilder-ai/botbuilder/ai/qna/models/train_request_body.py b/libraries/botbuilder-ai/botbuilder/ai/qna/models/train_request_body.py new file mode 100644 index 000000000..2ce267831 --- /dev/null +++ b/libraries/botbuilder-ai/botbuilder/ai/qna/models/train_request_body.py @@ -0,0 +1,27 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import List +from msrest.serialization import Model + +from .feedback_record import FeedbackRecord + + +class TrainRequestBody(Model): + """ Class the models the request body that is sent as feedback to the Train API. """ + + _attribute_map = { + "feedback_records": {"key": "feedbackRecords", "type": "[FeedbackRecord]"} + } + + def __init__(self, feedback_records: List[FeedbackRecord], **kwargs): + """ + Parameters: + ----------- + + feedback_records: List of feedback records. + """ + + super().__init__(**kwargs) + + self.feedback_records = feedback_records diff --git a/libraries/botbuilder-ai/botbuilder/ai/qna/qnamaker.py b/libraries/botbuilder-ai/botbuilder/ai/qna/qnamaker.py index adbd6256a..cacdd7b79 100644 --- a/libraries/botbuilder-ai/botbuilder/ai/qna/qnamaker.py +++ b/libraries/botbuilder-ai/botbuilder/ai/qna/qnamaker.py @@ -1,28 +1,26 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from copy import copy import json -import platform -from typing import Dict, NamedTuple, Union +from typing import Dict, List, NamedTuple, Union +from aiohttp import ClientSession, ClientTimeout -from aiohttp import ClientSession, ClientTimeout, ClientResponse from botbuilder.schema import Activity from botbuilder.core import BotTelemetryClient, NullTelemetryClient, TurnContext -from .query_result import QueryResult +from .models import FeedbackRecord, QueryResult +from .utils import ( + ActiveLearningUtils, + GenerateAnswerUtils, + QnATelemetryConstants, + TrainUtils, +) from .qnamaker_endpoint import QnAMakerEndpoint from .qnamaker_options import QnAMakerOptions from .qnamaker_telemetry_client import QnAMakerTelemetryClient -from .qna_telemetry_constants import QnATelemetryConstants -from .qnamaker_trace_info import QnAMakerTraceInfo from .. import __title__, __version__ -QNAMAKER_TRACE_NAME = "QnAMaker" -QNAMAKER_TRACE_LABEL = "QnAMaker Trace" -QNAMAKER_TRACE_TYPE = "https://www.qnamaker.ai/schemas/trace" - class EventData(NamedTuple): properties: Dict[str, str] @@ -34,7 +32,7 @@ class QnAMaker(QnAMakerTelemetryClient): Class used to query a QnA Maker knowledge base for answers. """ - def __init__( # pylint: disable=super-init-not-called + def __init__( self, endpoint: QnAMakerEndpoint, options: QnAMakerOptions = None, @@ -42,6 +40,8 @@ def __init__( # pylint: disable=super-init-not-called telemetry_client: BotTelemetryClient = None, log_personal_information: bool = None, ): + super().__init__(log_personal_information, telemetry_client) + if not isinstance(endpoint, QnAMakerEndpoint): raise TypeError( "QnAMaker.__init__(): endpoint is not an instance of QnAMakerEndpoint" @@ -54,19 +54,84 @@ def __init__( # pylint: disable=super-init-not-called ) self._endpoint: str = endpoint - self._is_legacy_protocol: bool = self._endpoint.host.endswith("v3.0") - self._options = options or QnAMakerOptions() - self._validate_options(self._options) + opt = options or QnAMakerOptions() + self._validate_options(opt) - instance_timeout = ClientTimeout(total=self._options.timeout / 1000) - self._req_client = http_client or ClientSession(timeout=instance_timeout) + instance_timeout = ClientTimeout(total=opt.timeout / 1000) + self._http_client = http_client or ClientSession(timeout=instance_timeout) self.telemetry_client: Union[ BotTelemetryClient, NullTelemetryClient ] = telemetry_client or NullTelemetryClient() + self.log_personal_information = log_personal_information or False + self._generate_answer_helper = GenerateAnswerUtils( + self.telemetry_client, self._endpoint, options, self._http_client + ) + self._active_learning_train_helper = TrainUtils( + self._endpoint, self._http_client + ) + + async def get_answers( + self, + context: TurnContext, + options: QnAMakerOptions = None, + telemetry_properties: Dict[str, str] = None, + telemetry_metrics: Dict[str, int] = None, + ) -> [QueryResult]: + """ + Generates answers from the knowledge base. + + return: + ------- + A list of answers for the user's query, sorted in decreasing order of ranking score. + + rtype: + ------ + List[QueryResult] + """ + if not context: + raise TypeError("QnAMaker.get_answers(): context cannot be None.") + + if not isinstance(context.activity, Activity): + raise TypeError( + "QnAMaker.get_answers(): TurnContext's activity must be an Activity instance." + ) + + result = await self._generate_answer_helper.get_answers(context, options) + + await self.on_qna_result( + result, context, telemetry_properties, telemetry_metrics + ) + + return result + + def get_low_score_variation(self, query_result: QueryResult) -> List[QueryResult]: + """ + Filters the ambiguous question for active learning. + + Parameters: + ----------- + query_result: User query output. + + Return: + ------- + Filtered aray of ambigous questions. + """ + return ActiveLearningUtils.get_low_score_variation(query_result) + + async def call_train(self, feedback_records: List[FeedbackRecord]): + """ + Sends feedback to the knowledge base. + + Parameters: + ----------- + feedback_records + """ + return await self._active_learning_train_helper.call_train(feedback_records) + async def on_qna_result( self, query_results: [QueryResult], @@ -78,6 +143,7 @@ async def on_qna_result( query_results, turn_context, telemetry_properties, telemetry_metrics ) + # Track the event self.telemetry_client.track_event( name=QnATelemetryConstants.qna_message_event, properties=event_data.properties, @@ -93,10 +159,17 @@ async def fill_qna_event( ) -> EventData: """ Fills the event properties and metrics for the QnaMessage event for telemetry. - :return: A tuple of event data properties and metrics that will be sent to the BotTelemetryClient.track_event() - method for the QnAMessage event. The properties and metrics returned the standard properties logged with any - properties passed from the get_answers() method. - :rtype: EventData + + return: + ------- + A tuple of event data properties and metrics that will be sent to the + BotTelemetryClient.track_event() method for the QnAMessage event. + The properties and metrics returned the standard properties logged + with any properties passed from the get_answers() method. + + rtype: + ------ + EventData """ properties: Dict[str, str] = dict() @@ -118,7 +191,7 @@ async def fill_qna_event( properties[QnATelemetryConstants.username_property] = user_name # Fill in Qna Results (found or not). - if query_results: + if self._has_matched_answer_in_kb(query_results): query_result = query_results[0] result_properties = { @@ -152,32 +225,6 @@ async def fill_qna_event( return EventData(properties=properties, metrics=metrics) - async def get_answers( - self, - context: TurnContext, - options: QnAMakerOptions = None, - telemetry_properties: Dict[str, str] = None, - telemetry_metrics: Dict[str, int] = None, - ) -> [QueryResult]: - """ - Generates answers from the knowledge base. - :return: A list of answers for the user's query, sorted in decreasing order of ranking score. - :rtype: [QueryResult] - """ - - hydrated_options = self._hydrate_options(options) - self._validate_options(hydrated_options) - - result = await self._query_qna_service(context, hydrated_options) - - await self.on_qna_result( - result, context, telemetry_properties, telemetry_metrics - ) - - await self._emit_trace_info(context, result, hydrated_options) - - return result - def _validate_options(self, options: QnAMakerOptions): if not options.score_threshold: options.score_threshold = 0.3 @@ -197,132 +244,10 @@ def _validate_options(self, options: QnAMakerOptions): if not options.timeout: options.timeout = 100000 - def _hydrate_options(self, query_options: QnAMakerOptions) -> QnAMakerOptions: - """ - Combines QnAMakerOptions passed into the QnAMaker constructor with the options passed as arguments - into get_answers(). - :return: QnAMakerOptions with options passed into constructor overwritten by new options passed into - get_answers() - :rtype: QnAMakerOptions - """ - - hydrated_options = copy(self._options) - - if query_options: - if ( - query_options.score_threshold != hydrated_options.score_threshold - and query_options.score_threshold - ): - hydrated_options.score_threshold = query_options.score_threshold - - if query_options.top != hydrated_options.top and query_options.top != 0: - hydrated_options.top = query_options.top - - if query_options.strict_filters: - hydrated_options.strict_filters = query_options.strict_filters - - if ( - query_options.timeout != hydrated_options.timeout - and query_options.timeout - ): - hydrated_options.timeout = query_options.timeout - - return hydrated_options - - async def _query_qna_service( - self, turn_context: TurnContext, options: QnAMakerOptions - ) -> [QueryResult]: - url = f"{ self._endpoint.host }/knowledgebases/{ self._endpoint.knowledge_base_id }/generateAnswer" - question = { - "question": turn_context.activity.text, - "top": options.top, - "scoreThreshold": options.score_threshold, - "strictFilters": options.strict_filters, - } - serialized_content = json.dumps(question) - headers = self._get_headers() - - # Convert miliseconds to seconds (as other BotBuilder SDKs accept timeout value in miliseconds) - # aiohttp.ClientSession units are in seconds - timeout = ClientTimeout(total=options.timeout / 1000) - - response = await self._req_client.post( - url, data=serialized_content, headers=headers, timeout=timeout - ) - - result = await self._format_qna_result(response, options) - - return result - - async def _emit_trace_info( - self, turn_context: TurnContext, result: [QueryResult], options: QnAMakerOptions - ): - trace_info = QnAMakerTraceInfo( - message=turn_context.activity, - query_results=result, - knowledge_base_id=self._endpoint.knowledge_base_id, - score_threshold=options.score_threshold, - top=options.top, - strict_filters=options.strict_filters, - ) - - trace_activity = Activity( - label=QNAMAKER_TRACE_LABEL, - name=QNAMAKER_TRACE_NAME, - type="trace", - value=trace_info, - value_type=QNAMAKER_TRACE_TYPE, - ) - - await turn_context.send_activity(trace_activity) - - async def _format_qna_result( - self, result, options: QnAMakerOptions - ) -> [QueryResult]: - json_res = result - if isinstance(result, ClientResponse): - json_res = await result.json() - - answers_within_threshold = [ - {**answer, "score": answer["score"] / 100} - for answer in json_res["answers"] - if answer["score"] / 100 > options.score_threshold - ] - sorted_answers = sorted( - answers_within_threshold, key=lambda ans: ans["score"], reverse=True - ) - - # The old version of the protocol returns the id in a field called qnaId - # The following translates this old structure to the new - if self._is_legacy_protocol: - for answer in answers_within_threshold: - answer["id"] = answer.pop("qnaId", None) - - answers_as_query_results = list( - map(lambda answer: QueryResult(**answer), sorted_answers) - ) - - return answers_as_query_results - - def _get_headers(self): - headers = { - "Content-Type": "application/json", - "User-Agent": self.get_user_agent(), - } - - if self._is_legacy_protocol: - headers["Ocp-Apim-Subscription-Key"] = self._endpoint.endpoint_key - else: - headers["Authorization"] = f"EndpointKey {self._endpoint.endpoint_key}" - - return headers + def _has_matched_answer_in_kb(self, query_results: [QueryResult]) -> bool: + if query_results: + if query_results[0].id != -1: - def get_user_agent(self): - package_user_agent = f"{__title__}/{__version__}" - uname = platform.uname() - os_version = f"{uname.machine}-{uname.system}-{uname.version}" - py_version = f"Python,Version={platform.python_version()}" - platform_user_agent = f"({os_version}; {py_version})" - user_agent = f"{package_user_agent} {platform_user_agent}" + return True - return user_agent + return False diff --git a/libraries/botbuilder-ai/botbuilder/ai/qna/qnamaker_options.py b/libraries/botbuilder-ai/botbuilder/ai/qna/qnamaker_options.py index 42cf0f0ce..6387a4682 100644 --- a/libraries/botbuilder-ai/botbuilder/ai/qna/qnamaker_options.py +++ b/libraries/botbuilder-ai/botbuilder/ai/qna/qnamaker_options.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from .metadata import Metadata +from .models import Metadata # figure out if 300 milliseconds is ok for python requests library...or 100000 diff --git a/libraries/botbuilder-ai/botbuilder/ai/qna/qnamaker_trace_info.py b/libraries/botbuilder-ai/botbuilder/ai/qna/qnamaker_trace_info.py deleted file mode 100644 index 45f970ba0..000000000 --- a/libraries/botbuilder-ai/botbuilder/ai/qna/qnamaker_trace_info.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -from botbuilder.schema import Activity -from .query_result import QueryResult - -# Should we set the options=None in TraceInfo? (not optional in node) -class QnAMakerTraceInfo: - def __init__( - self, - message: Activity, - query_results: [QueryResult], - knowledge_base_id, - score_threshold, - top, - strict_filters, - ): - self.message = (message,) - self.query_results = (query_results,) - self.knowledge_base_id = (knowledge_base_id,) - self.score_threshold = (score_threshold,) - self.top = (top,) - self.strict_filters = strict_filters diff --git a/libraries/botbuilder-ai/botbuilder/ai/qna/query_result.py b/libraries/botbuilder-ai/botbuilder/ai/qna/query_result.py deleted file mode 100644 index f1a13ec0b..000000000 --- a/libraries/botbuilder-ai/botbuilder/ai/qna/query_result.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -from .metadata import Metadata - - -class QueryResult: - def __init__( - self, - questions: [str], - answer: str, - score: float, - metadata: [Metadata], - source: str, - id: int, # pylint: disable=invalid-name - context=None, - ): - self.questions = (questions,) - self.answer = (answer,) - self.score = (score,) - self.metadata = (list(map(lambda meta: Metadata(**meta), metadata)),) - self.source = source - self.id = id # pylint: disable=invalid-name - - # 4.4 multi-turn - self.context = context diff --git a/libraries/botbuilder-ai/botbuilder/ai/qna/utils/__init__.py b/libraries/botbuilder-ai/botbuilder/ai/qna/utils/__init__.py new file mode 100644 index 000000000..58d8575e0 --- /dev/null +++ b/libraries/botbuilder-ai/botbuilder/ai/qna/utils/__init__.py @@ -0,0 +1,20 @@ +# coding=utf-8 +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +from .active_learning_utils import ActiveLearningUtils +from .generate_answer_utils import GenerateAnswerUtils +from .http_request_utils import HttpRequestUtils +from .qna_telemetry_constants import QnATelemetryConstants +from .train_utils import TrainUtils + +__all__ = [ + "ActiveLearningUtils", + "GenerateAnswerUtils", + "HttpRequestUtils", + "QnATelemetryConstants", + "TrainUtils", +] diff --git a/libraries/botbuilder-ai/botbuilder/ai/qna/utils/active_learning_utils.py b/libraries/botbuilder-ai/botbuilder/ai/qna/utils/active_learning_utils.py new file mode 100644 index 000000000..4fc6c536f --- /dev/null +++ b/libraries/botbuilder-ai/botbuilder/ai/qna/utils/active_learning_utils.py @@ -0,0 +1,65 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import math + +from typing import List +from ..models import QueryResult + +MINIMUM_SCORE_FOR_LOW_SCORE_VARIATION = 20.0 +PREVIOUS_LOW_SCORE_VARIATION_MULTIPLIER = 1.4 +MAX_LOW_SCORE_VARIATION_MULTIPLIER = 2.0 +MAX_SCORE_FOR_LOW_SCORE_VARIATION = 95.0 + + +class ActiveLearningUtils: + """ Active learning helper class """ + + @staticmethod + def get_low_score_variation( + qna_search_results: List[QueryResult] + ) -> List[QueryResult]: + """ + Returns a list of QnA search results, which have low score variation. + + Parameters: + ----------- + + qna_serach_results: A list of QnA QueryResults returned from the QnA GenerateAnswer API call. + """ + + if not qna_search_results: + return [] + + if len(qna_search_results) == 1: + return qna_search_results + + filtered_qna_search_result: List[QueryResult] = [] + top_answer_score = qna_search_results[0].score * 100 + prev_score = top_answer_score + + if ( + MINIMUM_SCORE_FOR_LOW_SCORE_VARIATION + < top_answer_score + <= MAX_SCORE_FOR_LOW_SCORE_VARIATION + ): + filtered_qna_search_result.append(qna_search_results[0]) + + for idx in range(1, len(qna_search_results)): + current_score = qna_search_results[idx].score * 100 + + if ActiveLearningUtils._include_for_clustering( + prev_score, current_score, PREVIOUS_LOW_SCORE_VARIATION_MULTIPLIER + ) and ActiveLearningUtils._include_for_clustering( + top_answer_score, current_score, MAX_LOW_SCORE_VARIATION_MULTIPLIER + ): + prev_score = current_score + filtered_qna_search_result.append(qna_search_results[idx]) + + return filtered_qna_search_result + + @staticmethod + def _include_for_clustering( + prev_score: float, current_score: float, multiplier: float + ) -> bool: + return (prev_score - current_score) < (multiplier * math.sqrt(prev_score)) diff --git a/libraries/botbuilder-ai/botbuilder/ai/qna/utils/generate_answer_utils.py b/libraries/botbuilder-ai/botbuilder/ai/qna/utils/generate_answer_utils.py new file mode 100644 index 000000000..8651fb1ed --- /dev/null +++ b/libraries/botbuilder-ai/botbuilder/ai/qna/utils/generate_answer_utils.py @@ -0,0 +1,205 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from copy import copy +from typing import List, Union + +from aiohttp import ClientResponse, ClientSession + +from botbuilder.core import BotTelemetryClient, NullTelemetryClient, TurnContext +from botbuilder.schema import Activity + +from .http_request_utils import HttpRequestUtils + +from ..qnamaker_endpoint import QnAMakerEndpoint +from ..qnamaker_options import QnAMakerOptions +from ..models import GenerateAnswerRequestBody, QnAMakerTraceInfo, QueryResult + +QNAMAKER_TRACE_NAME = "QnAMaker" +QNAMAKER_TRACE_LABEL = "QnAMaker Trace" +QNAMAKER_TRACE_TYPE = "https://www.qnamaker.ai/schemas/trace" + + +class GenerateAnswerUtils: + """ + Helper class for Generate Answer API, which is used to make queries to + a single QnA Maker knowledge base and return the result. + """ + + def __init__( + self, + telemetry_client: Union[BotTelemetryClient, NullTelemetryClient], + endpoint: QnAMakerEndpoint, + options: QnAMakerOptions, + http_client: ClientSession, + ): + """ + Parameters: + ----------- + + telemetry_client: Telemetry client. + + endpoint: QnA Maker endpoint details. + + options: QnA Maker options to configure the instance. + + http_client: HTTP client. + """ + self._telemetry_client = telemetry_client + self._endpoint = endpoint + + self.options = ( + options if isinstance(options, QnAMakerOptions) else QnAMakerOptions() + ) + self._validate_options(self.options) + + self._http_client = http_client + + async def get_answers( + self, context: TurnContext, options: QnAMakerOptions = None + ) -> List[QueryResult]: + if not isinstance(context, TurnContext): + raise TypeError( + "GenerateAnswerUtils.get_answers(): context must be an instance of TurnContext" + ) + + hydrated_options = self._hydrate_options(options) + self._validate_options(hydrated_options) + + result: List[QueryResult] = await self._query_qna_service( + context, hydrated_options + ) + + await self._emit_trace_info(context, result, hydrated_options) + + return result + + def _validate_options(self, options: QnAMakerOptions): + if not options.score_threshold: + options.score_threshold = 0.3 + + if not options.top: + options.top = 1 + + if options.score_threshold < 0 or options.score_threshold > 1: + raise ValueError("Score threshold should be a value between 0 and 1") + + if options.top < 1: + raise ValueError("QnAMakerOptions.top should be an integer greater than 0") + + if not options.strict_filters: + options.strict_filters = [] + + if not options.timeout: + options.timeout = 100000 + + def _hydrate_options(self, query_options: QnAMakerOptions) -> QnAMakerOptions: + """ + Combines QnAMakerOptions passed into the QnAMaker constructor + with the options passed as arguments into get_answers(). + Return: + ------- + QnAMakerOptions with options passed into constructor overwritten by new options passed into get_answers() + + rtype: + ------ + QnAMakerOptions + """ + + hydrated_options = copy(self.options) + + if query_options: + if ( + query_options.score_threshold != hydrated_options.score_threshold + and query_options.score_threshold + ): + hydrated_options.score_threshold = query_options.score_threshold + + if query_options.top != hydrated_options.top and query_options.top != 0: + hydrated_options.top = query_options.top + + if query_options.strict_filters: + hydrated_options.strict_filters = query_options.strict_filters + + if ( + query_options.timeout != hydrated_options.timeout + and query_options.timeout + ): + hydrated_options.timeout = query_options.timeout + + return hydrated_options + + async def _query_qna_service( + self, context: TurnContext, options: QnAMakerOptions + ) -> List[QueryResult]: + url = f"{ self._endpoint.host }/knowledgebases/{ self._endpoint.knowledge_base_id }/generateAnswer" + + question = GenerateAnswerRequestBody( + question=context.activity.text, + top=options.top, + score_threshold=options.score_threshold, + strict_filters=options.strict_filters, + ) + + http_request_helper = HttpRequestUtils(self._http_client) + + response: ClientResponse = await http_request_helper.execute_http_request( + url, question, self._endpoint, options.timeout + ) + + result: List[QueryResult] = await self._format_qna_result(response, options) + + return result + + async def _emit_trace_info( + self, context: TurnContext, result: List[QueryResult], options: QnAMakerOptions + ): + trace_info = QnAMakerTraceInfo( + message=context.activity, + query_results=result, + knowledge_base_id=self._endpoint.knowledge_base_id, + score_threshold=options.score_threshold, + top=options.top, + strict_filters=options.strict_filters, + ) + + trace_activity = Activity( + label=QNAMAKER_TRACE_LABEL, + name=QNAMAKER_TRACE_NAME, + type="trace", + value=trace_info, + value_type=QNAMAKER_TRACE_TYPE, + ) + + await context.send_activity(trace_activity) + + async def _format_qna_result( + self, result, options: QnAMakerOptions + ) -> List[QueryResult]: + json_res = result + if isinstance(result, ClientResponse): + json_res = await result.json() + + answers_within_threshold = [ + {**answer, "score": answer["score"] / 100} + for answer in json_res["answers"] + if answer["score"] / 100 > options.score_threshold + ] + sorted_answers = sorted( + answers_within_threshold, key=lambda ans: ans["score"], reverse=True + ) + + # The old version of the protocol returns the id in a field called qnaId + # The following translates this old structure to the new + is_legacy_protocol: bool = self._endpoint.host.endswith( + "v2.0" + ) or self._endpoint.host.endswith("v3.0") + if is_legacy_protocol: + for answer in answers_within_threshold: + answer["id"] = answer.pop("qnaId", None) + + answers_as_query_results = list( + map(lambda answer: QueryResult(**answer), sorted_answers) + ) + + return answers_as_query_results diff --git a/libraries/botbuilder-ai/botbuilder/ai/qna/utils/http_request_utils.py b/libraries/botbuilder-ai/botbuilder/ai/qna/utils/http_request_utils.py new file mode 100644 index 000000000..d550f8ad0 --- /dev/null +++ b/libraries/botbuilder-ai/botbuilder/ai/qna/utils/http_request_utils.py @@ -0,0 +1,95 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import json +import platform + +from aiohttp import ClientResponse, ClientSession, ClientTimeout + +from ... import __title__, __version__ + +from ..qnamaker_endpoint import QnAMakerEndpoint + + +class HttpRequestUtils: + """ HTTP request utils class. """ + + def __init__(self, http_client: ClientSession): + self._http_client = http_client + + async def execute_http_request( + self, + request_url: str, + payload_body: object, + endpoint: QnAMakerEndpoint, + timeout: float = None, + ) -> ClientResponse: + """ + Execute HTTP request. + + Parameters: + ----------- + + request_url: HTTP request URL. + + payload_body: HTTP request body. + + endpoint: QnA Maker endpoint details. + + timeout: Timeout for HTTP call (milliseconds). + """ + if not request_url: + raise TypeError( + "HttpRequestUtils.execute_http_request(): request_url cannot be None." + ) + + if not payload_body: + raise TypeError( + "HttpRequestUtils.execute_http_request(): question cannot be None." + ) + + if not endpoint: + raise TypeError( + "HttpRequestUtils.execute_http_request(): endpoint cannot be None." + ) + + serialized_payload_body = json.dumps(payload_body.serialize()) + + headers = self._get_headers(endpoint) + + if timeout: + # Convert miliseconds to seconds (as other BotBuilder SDKs accept timeout value in miliseconds) + # aiohttp.ClientSession units are in seconds + request_timeout = ClientTimeout(total=timeout / 1000) + + response: ClientResponse = await self._http_client.post( + request_url, + data=serialized_payload_body, + headers=headers, + timeout=request_timeout, + ) + else: + response: ClientResponse = await self._http_client.post( + request_url, data=serialized_payload_body, headers=headers + ) + + return response + + def _get_headers(self, endpoint: QnAMakerEndpoint): + headers = { + "Content-Type": "application/json", + "User-Agent": self._get_user_agent(), + "Authorization": f"EndpointKey {endpoint.endpoint_key}", + } + + return headers + + def _get_user_agent(self): + package_user_agent = f"{__title__}/{__version__}" + uname = platform.uname() + os_version = f"{uname.machine}-{uname.system}-{uname.version}" + py_version = f"Python,Version={platform.python_version()}" + platform_user_agent = f"({os_version}; {py_version})" + user_agent = f"{package_user_agent} {platform_user_agent}" + + return user_agent diff --git a/libraries/botbuilder-ai/botbuilder/ai/qna/qna_telemetry_constants.py b/libraries/botbuilder-ai/botbuilder/ai/qna/utils/qna_telemetry_constants.py similarity index 89% rename from libraries/botbuilder-ai/botbuilder/ai/qna/qna_telemetry_constants.py rename to libraries/botbuilder-ai/botbuilder/ai/qna/utils/qna_telemetry_constants.py index 37ffe8e17..8588b343e 100644 --- a/libraries/botbuilder-ai/botbuilder/ai/qna/qna_telemetry_constants.py +++ b/libraries/botbuilder-ai/botbuilder/ai/qna/utils/qna_telemetry_constants.py @@ -6,7 +6,7 @@ class QnATelemetryConstants(str, Enum): """ - The IBotTelemetryClient event and property names that logged by default. + Default QnA event and property names logged using IBotTelemetryClient. """ qna_message_event = "QnaMessage" diff --git a/libraries/botbuilder-ai/botbuilder/ai/qna/utils/train_utils.py b/libraries/botbuilder-ai/botbuilder/ai/qna/utils/train_utils.py new file mode 100644 index 000000000..ec2586bd4 --- /dev/null +++ b/libraries/botbuilder-ai/botbuilder/ai/qna/utils/train_utils.py @@ -0,0 +1,54 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from typing import List +from aiohttp import ClientSession + +from ..qnamaker_endpoint import QnAMakerEndpoint +from ..models import FeedbackRecord, TrainRequestBody + +from .http_request_utils import HttpRequestUtils + + +class TrainUtils: + """ Class for Train API, used in active learning to add suggestions to the knowledge base """ + + def __init__(self, endpoint: QnAMakerEndpoint, http_client: ClientSession): + """ + Initializes a new instance for active learning train utils. + + Parameters: + ----------- + + endpoint: QnA Maker Endpoint of the knowledge base to query. + + http_client: Http client. + """ + self._endpoint = endpoint + self._http_client = http_client + + async def call_train(self, feedback_records: List[FeedbackRecord]): + """ + Train API to provide feedback. + + Parameter: + ------------- + + feedback_records: Feedback record list. + """ + if not feedback_records: + raise TypeError("TrainUtils.call_train(): feedback_records cannot be None.") + + if not feedback_records: + return + + await self._query_train(feedback_records) + + async def _query_train(self, feedback_records: List[FeedbackRecord]): + url: str = f"{ self._endpoint.host }/knowledgebases/{ self._endpoint.knowledge_base_id }/train" + payload_body = TrainRequestBody(feedback_records=feedback_records) + http_request_helper = HttpRequestUtils(self._http_client) + + await http_request_helper.execute_http_request( + url, payload_body, self._endpoint + ) diff --git a/libraries/botbuilder-ai/tests/qna/test_data/TopNAnswer.json b/libraries/botbuilder-ai/tests/qna/test_data/TopNAnswer.json new file mode 100644 index 000000000..a5ace2bd5 --- /dev/null +++ b/libraries/botbuilder-ai/tests/qna/test_data/TopNAnswer.json @@ -0,0 +1,64 @@ +{ + "answers": [ + { + "questions": [ + "Q1" + ], + "answer": "A1", + "score": 80, + "id": 15, + "source": "Editorial", + "metadata": [ + { + "name": "topic", + "value": "value" + } + ] + }, + { + "questions": [ + "Q2" + ], + "answer": "A2", + "score": 78, + "id": 16, + "source": "Editorial", + "metadata": [ + { + "name": "topic", + "value": "value" + } + ] + }, + { + "questions": [ + "Q3" + ], + "answer": "A3", + "score": 75, + "id": 17, + "source": "Editorial", + "metadata": [ + { + "name": "topic", + "value": "value" + } + ] + }, + { + "questions": [ + "Q4" + ], + "answer": "A4", + "score": 50, + "id": 18, + "source": "Editorial", + "metadata": [ + { + "name": "topic", + "value": "value" + } + ] + } + ] + } \ No newline at end of file diff --git a/libraries/botbuilder-ai/tests/qna/test_qna.py b/libraries/botbuilder-ai/tests/qna/test_qna.py index 77314316f..148041ec1 100644 --- a/libraries/botbuilder-ai/tests/qna/test_qna.py +++ b/libraries/botbuilder-ai/tests/qna/test_qna.py @@ -11,14 +11,9 @@ from aiohttp import ClientSession import aiounittest -from botbuilder.ai.qna import ( - Metadata, - QnAMakerEndpoint, - QnAMaker, - QnAMakerOptions, - QnATelemetryConstants, - QueryResult, -) +from botbuilder.ai.qna import QnAMakerEndpoint, QnAMaker, QnAMakerOptions +from botbuilder.ai.qna.models import FeedbackRecord, Metadata, QueryResult +from botbuilder.ai.qna.utils import QnATelemetryConstants from botbuilder.core import BotAdapter, BotTelemetryClient, TurnContext from botbuilder.core.adapters import TestAdapter from botbuilder.schema import ( @@ -44,8 +39,9 @@ async def capture_sent_activities( class QnaApplicationTest(aiounittest.AsyncTestCase): - # Note this is NOT a real LUIS application ID nor a real LUIS subscription-key + # Note this is NOT a real QnA Maker application ID nor a real QnA Maker subscription-key # theses are GUIDs edited to look right to the parsing and validation code. + _knowledge_base_id: str = "f028d9k3-7g9z-11d3-d300-2b8x98227q8w" _endpoint_key: str = "1k997n7w-207z-36p3-j2u1-09tas20ci6011" _host: str = "https://dummyqnahost.azurewebsites.net/qnamaker" @@ -89,37 +85,10 @@ def test_qnamaker_with_none_endpoint(self): with self.assertRaises(TypeError): QnAMaker(None) - def test_v2_legacy_endpoint(self): - v2_hostname = "https://westus.api.cognitive.microsoft.com/qnamaker/v2.0" - - v2_legacy_endpoint = QnAMakerEndpoint( - self._knowledge_base_id, self._endpoint_key, v2_hostname - ) - - with self.assertRaises(ValueError): - QnAMaker(v2_legacy_endpoint) - - def test_legacy_protocol(self): - v3_hostname = "https://westus.api.cognitive.microsoft.com/qnamaker/v3.0" - v3_legacy_endpoint = QnAMakerEndpoint( - self._knowledge_base_id, self._endpoint_key, v3_hostname - ) - legacy_qna = QnAMaker(v3_legacy_endpoint) - is_legacy = True - - v4_hostname = "https://UpdatedNonLegacyQnaHostName.azurewebsites.net/qnamaker" - nonlegacy_endpoint = QnAMakerEndpoint( - self._knowledge_base_id, self._endpoint_key, v4_hostname - ) - v4_qna = QnAMaker(nonlegacy_endpoint) - - self.assertEqual(is_legacy, legacy_qna._is_legacy_protocol) - self.assertNotEqual(is_legacy, v4_qna._is_legacy_protocol) - def test_set_default_options_with_no_options_arg(self): qna_without_options = QnAMaker(self.tests_endpoint) - options = qna_without_options._options + options = qna_without_options._generate_answer_helper.options default_threshold = 0.3 default_top = 1 @@ -138,7 +107,7 @@ def test_options_passed_to_ctor(self): ) qna_with_options = QnAMaker(self.tests_endpoint, options) - actual_options = qna_with_options._options + actual_options = qna_with_options._generate_answer_helper.options expected_threshold = 0.8 expected_timeout = 9000 @@ -170,7 +139,7 @@ async def test_returns_answer(self): self.assertEqual(1, len(result)) self.assertEqual( "BaseCamp: You can use a damp rag to clean around the Power Pack", - first_answer.answer[0], + first_answer.answer, ) async def test_returns_answer_using_options(self): @@ -178,9 +147,7 @@ async def test_returns_answer_using_options(self): question: str = "up" response_path: str = "AnswerWithOptions.json" options = QnAMakerOptions( - score_threshold=0.8, - top=5, - strict_filters=[{"name": "movie", "value": "disney"}], + score_threshold=0.8, top=5, strict_filters=[Metadata("movie", "disney")] ) # Act @@ -190,14 +157,14 @@ async def test_returns_answer_using_options(self): first_answer = result[0] has_at_least_1_ans = True - first_metadata = first_answer.metadata[0][0] + first_metadata = first_answer.metadata[0] # Assert self.assertIsNotNone(result) self.assertEqual(has_at_least_1_ans, len(result) >= 1) self.assertTrue(first_answer.answer[0]) - self.assertEqual("is a movie", first_answer.answer[0]) - self.assertTrue(first_answer.score[0] >= options.score_threshold) + self.assertEqual("is a movie", first_answer.answer) + self.assertTrue(first_answer.score >= options.score_threshold) self.assertEqual("movie", first_metadata.name) self.assertEqual("disney", first_metadata.value) @@ -242,7 +209,7 @@ async def test_trace_test(self): self.assertEqual(True, hasattr(trace_activity.value, "top")) self.assertEqual(True, hasattr(trace_activity.value, "strict_filters")) self.assertEqual( - self._knowledge_base_id, trace_activity.value.knowledge_base_id[0] + self._knowledge_base_id, trace_activity.value.knowledge_base_id ) return result @@ -261,7 +228,9 @@ async def test_returns_answer_with_timeout(self): result = await qna.get_answers(context, options) self.assertIsNotNone(result) - self.assertEqual(options.timeout, qna._options.timeout) + self.assertEqual( + options.timeout, qna._generate_answer_helper.options.timeout + ) async def test_telemetry_returns_answer(self): # Arrange @@ -289,7 +258,7 @@ async def test_telemetry_returns_answer(self): number_of_args = len(telemetry_args) first_answer = telemetry_args["properties"][ QnATelemetryConstants.answer_property - ][0] + ] expected_answer = ( "BaseCamp: You can use a damp rag to clean around the Power Pack" ) @@ -306,12 +275,12 @@ async def test_telemetry_returns_answer(self): self.assertTrue("articleFound" in telemetry_properties) self.assertEqual(expected_answer, first_answer) self.assertTrue("score" in telemetry_metrics) - self.assertEqual(1, telemetry_metrics["score"][0]) + self.assertEqual(1, telemetry_metrics["score"]) # Assert - Validate we didn't break QnA functionality. self.assertIsNotNone(results) self.assertEqual(1, len(results)) - self.assertEqual(expected_answer, results[0].answer[0]) + self.assertEqual(expected_answer, results[0].answer) self.assertEqual("Editorial", results[0].source) async def test_telemetry_returns_answer_when_no_answer_found_in_kb(self): @@ -388,7 +357,7 @@ async def test_telemetry_pii(self): number_of_args = len(telemetry_args) first_answer = telemetry_args["properties"][ QnATelemetryConstants.answer_property - ][0] + ] expected_answer = ( "BaseCamp: You can use a damp rag to clean around the Power Pack" ) @@ -405,12 +374,12 @@ async def test_telemetry_pii(self): self.assertTrue("articleFound" in telemetry_properties) self.assertEqual(expected_answer, first_answer) self.assertTrue("score" in telemetry_metrics) - self.assertEqual(1, telemetry_metrics["score"][0]) + self.assertEqual(1, telemetry_metrics["score"]) # Assert - Validate we didn't break QnA functionality. self.assertIsNotNone(results) self.assertEqual(1, len(results)) - self.assertEqual(expected_answer, results[0].answer[0]) + self.assertEqual(expected_answer, results[0].answer) self.assertEqual("Editorial", results[0].source) async def test_telemetry_override(self): @@ -467,7 +436,7 @@ async def test_telemetry_override(self): # Validate we didn't break QnA functionality. self.assertIsNotNone(results) self.assertEqual(1, len(results)) - self.assertEqual(expected_answer, results[0].answer[0]) + self.assertEqual(expected_answer, results[0].answer) self.assertEqual("Editorial", results[0].source) async def test_telemetry_additional_props_metrics(self): @@ -515,7 +484,7 @@ async def test_telemetry_additional_props_metrics(self): self.assertTrue("matchedQuestion" in telemetry_properties) self.assertTrue("questionId" in telemetry_properties) self.assertTrue("answer" in telemetry_properties) - self.assertTrue(expected_answer, telemetry_properties["answer"][0]) + self.assertTrue(expected_answer, telemetry_properties["answer"]) self.assertTrue("my_important_property" in telemetry_properties) self.assertEqual( "my_important_value", telemetry_properties["my_important_property"] @@ -531,7 +500,7 @@ async def test_telemetry_additional_props_metrics(self): # Assert - Validate we didn't break QnA functionality. self.assertIsNotNone(results) self.assertEqual(1, len(results)) - self.assertEqual(expected_answer, results[0].answer[0]) + self.assertEqual(expected_answer, results[0].answer) self.assertEqual("Editorial", results[0].source) async def test_telemetry_additional_props_override(self): @@ -588,7 +557,7 @@ async def test_telemetry_additional_props_override(self): self.assertTrue("question" not in tracked_properties) self.assertTrue("questionId" in tracked_properties) self.assertTrue("answer" in tracked_properties) - self.assertEqual(expected_answer, tracked_properties["answer"][0]) + self.assertEqual(expected_answer, tracked_properties["answer"]) self.assertTrue("my_important_property" not in tracked_properties) self.assertEqual(1, len(tracked_metrics)) self.assertTrue("score" in tracked_metrics) @@ -597,7 +566,7 @@ async def test_telemetry_additional_props_override(self): # Assert - Validate we didn't break QnA functionality. self.assertIsNotNone(results) self.assertEqual(1, len(results)) - self.assertEqual(expected_answer, results[0].answer[0]) + self.assertEqual(expected_answer, results[0].answer) self.assertEqual("Editorial", results[0].source) async def test_telemetry_fill_props_override(self): @@ -657,7 +626,7 @@ async def test_telemetry_fill_props_override(self): self.assertEqual("my_important_value2", first_properties["matchedQuestion"]) self.assertTrue("questionId" in first_properties) self.assertTrue("answer" in first_properties) - self.assertEqual(expected_answer, first_properties["answer"][0]) + self.assertEqual(expected_answer, first_properties["answer"]) self.assertTrue("articleFound" in first_properties) self.assertTrue("my_important_property" in first_properties) self.assertEqual( @@ -671,9 +640,49 @@ async def test_telemetry_fill_props_override(self): # Assert - Validate we didn't break QnA functionality. self.assertIsNotNone(results) self.assertEqual(1, len(results)) - self.assertEqual(expected_answer, results[0].answer[0]) + self.assertEqual(expected_answer, results[0].answer) self.assertEqual("Editorial", results[0].source) + async def test_call_train(self): + feedback_records = [] + + feedback1 = FeedbackRecord( + qna_id=1, user_id="test", user_question="How are you?" + ) + + feedback2 = FeedbackRecord(qna_id=2, user_id="test", user_question="What up??") + + feedback_records.extend([feedback1, feedback2]) + + with patch.object( + QnAMaker, "call_train", return_value=None + ) as mocked_call_train: + qna = QnAMaker(QnaApplicationTest.tests_endpoint) + qna.call_train(feedback_records) + + mocked_call_train.assert_called_once_with(feedback_records) + + async def test_should_filter_low_score_variation(self): + options = QnAMakerOptions(top=5) + qna = QnAMaker(QnaApplicationTest.tests_endpoint, options) + question: str = "Q11" + context = QnaApplicationTest._get_context(question, TestAdapter()) + response_json = QnaApplicationTest._get_json_for_file("TopNAnswer.json") + + with patch( + "aiohttp.ClientSession.post", + return_value=aiounittest.futurized(response_json), + ): + results = await qna.get_answers(context) + self.assertEqual(4, len(results), "Should have received 4 answers.") + + filtered_results = qna.get_low_score_variation(results) + self.assertEqual( + 3, + len(filtered_results), + "Should have 3 filtered answers after low score variation.", + ) + @classmethod async def _get_service_result( cls, diff --git a/libraries/botbuilder-core/tests/test_turn_context.py b/libraries/botbuilder-core/tests/test_turn_context.py index d52da1766..bf1b0c758 100644 --- a/libraries/botbuilder-core/tests/test_turn_context.py +++ b/libraries/botbuilder-core/tests/test_turn_context.py @@ -110,9 +110,15 @@ async def update_activity_handler(context, activity, next_handler): assert new_context.adapter == old_adapter assert new_context.activity == old_activity assert new_context.responded is True - assert len(new_context._on_send_activities) == 1 - assert len(new_context._on_update_activity) == 1 - assert len(new_context._on_delete_activity) == 1 + assert ( + len(new_context._on_send_activities) == 1 + ) # pylint: disable=protected-access + assert ( + len(new_context._on_update_activity) == 1 + ) # pylint: disable=protected-access + assert ( + len(new_context._on_delete_activity) == 1 + ) # pylint: disable=protected-access def test_responded_should_be_automatically_set_to_false(self): context = TurnContext(SimpleAdapter(), ACTIVITY)