From 712bb7c3b18b5584fbb796b92aa26ef5b4ac41e1 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 22 Oct 2025 10:56:16 +0200 Subject: [PATCH 1/7] tests(huggingface): Support 1.0.0rc7 --- .../huggingface_hub/test_huggingface_hub.py | 621 ++++++++++-------- 1 file changed, 330 insertions(+), 291 deletions(-) diff --git a/tests/integrations/huggingface_hub/test_huggingface_hub.py b/tests/integrations/huggingface_hub/test_huggingface_hub.py index b9ab4df5bf..e5b08e6383 100644 --- a/tests/integrations/huggingface_hub/test_huggingface_hub.py +++ b/tests/integrations/huggingface_hub/test_huggingface_hub.py @@ -5,6 +5,7 @@ import httpx from huggingface_hub import InferenceClient +from huggingface_hub.inference._providers.hf_inference import HFInferenceConversational import sentry_sdk from sentry_sdk.utils import package_version @@ -616,58 +617,64 @@ def test_chat_completion( ) events = capture_events() - client = InferenceClient(model="test-model") - - with sentry_sdk.start_transaction(name="test"): - client.chat_completion( - messages=[{"role": "user", "content": "Hello!"}], - stream=False, - ) - - (transaction,) = events - - span = None - for sp in transaction["spans"]: - if sp["op"].startswith("gen_ai"): - assert span is None, "there is exactly one gen_ai span" - span = sp - else: - # there should be no other spans, just the gen_ai span - # and optionally some http.client spans from talking to the hf api - assert sp["op"] == "http.client" + def get_provider(*args, **kwargs): + return HFInferenceConversational() - assert span is not None + with mock.patch( + "huggingface_hub.inference._client.get_provider_helper", get_provider + ): + client = InferenceClient(model="test-model") - assert span["op"] == "gen_ai.chat" - assert span["description"] == "chat test-model" - assert span["origin"] == "auto.ai.huggingface_hub" + with sentry_sdk.start_transaction(name="test"): + client.chat_completion( + messages=[{"role": "user", "content": "Hello!"}], + stream=False, + ) - expected_data = { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "test-model", - "gen_ai.response.finish_reasons": "stop", - "gen_ai.response.model": "test-model-123", - "gen_ai.response.streaming": False, - "gen_ai.usage.input_tokens": 10, - "gen_ai.usage.output_tokens": 8, - "gen_ai.usage.total_tokens": 18, - "thread.id": mock.ANY, - "thread.name": mock.ANY, - } + (transaction,) = events + + span = None + for sp in transaction["spans"]: + if sp["op"].startswith("gen_ai"): + assert span is None, "there is exactly one gen_ai span" + span = sp + else: + # there should be no other spans, just the gen_ai span + # and optionally some http.client spans from talking to the hf api + assert sp["op"] == "http.client" + + assert span is not None + + assert span["op"] == "gen_ai.chat" + assert span["description"] == "chat test-model" + assert span["origin"] == "auto.ai.huggingface_hub" + + expected_data = { + "gen_ai.operation.name": "chat", + "gen_ai.request.model": "test-model", + "gen_ai.response.finish_reasons": "stop", + "gen_ai.response.model": "test-model-123", + "gen_ai.response.streaming": False, + "gen_ai.usage.input_tokens": 10, + "gen_ai.usage.output_tokens": 8, + "gen_ai.usage.total_tokens": 18, + "thread.id": mock.ANY, + "thread.name": mock.ANY, + } - if send_default_pii and include_prompts: - expected_data["gen_ai.request.messages"] = ( - '[{"role": "user", "content": "Hello!"}]' - ) - expected_data["gen_ai.response.text"] = ( - "[mocked] Hello! How can I help you today?" - ) + if send_default_pii and include_prompts: + expected_data["gen_ai.request.messages"] = ( + '[{"role": "user", "content": "Hello!"}]' + ) + expected_data["gen_ai.response.text"] = ( + "[mocked] Hello! How can I help you today?" + ) - if not send_default_pii or not include_prompts: - assert "gen_ai.request.messages" not in expected_data - assert "gen_ai.response.text" not in expected_data + if not send_default_pii or not include_prompts: + assert "gen_ai.request.messages" not in expected_data + assert "gen_ai.response.text" not in expected_data - assert span["data"] == expected_data + assert span["data"] == expected_data @pytest.mark.httpx_mock(assert_all_requests_were_expected=False) @@ -688,60 +695,66 @@ def test_chat_completion_streaming( ) events = capture_events() - client = InferenceClient(model="test-model") - - with sentry_sdk.start_transaction(name="test"): - _ = list( - client.chat_completion( - [{"role": "user", "content": "Hello!"}], - stream=True, - ) - ) - - (transaction,) = events - - span = None - for sp in transaction["spans"]: - if sp["op"].startswith("gen_ai"): - assert span is None, "there is exactly one gen_ai span" - span = sp - else: - # there should be no other spans, just the gen_ai span - # and optionally some http.client spans from talking to the hf api - assert sp["op"] == "http.client" - - assert span is not None + def get_provider(*args, **kwargs): + return HFInferenceConversational() - assert span["op"] == "gen_ai.chat" - assert span["description"] == "chat test-model" - assert span["origin"] == "auto.ai.huggingface_hub" + with mock.patch( + "huggingface_hub.inference._client.get_provider_helper", get_provider + ): + client = InferenceClient(model="test-model") - expected_data = { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "test-model", - "gen_ai.response.finish_reasons": "stop", - "gen_ai.response.model": "test-model-123", - "gen_ai.response.streaming": True, - "thread.id": mock.ANY, - "thread.name": mock.ANY, - } - # usage is not available in older versions of the library - if HF_VERSION and HF_VERSION >= (0, 26, 0): - expected_data["gen_ai.usage.input_tokens"] = 183 - expected_data["gen_ai.usage.output_tokens"] = 14 - expected_data["gen_ai.usage.total_tokens"] = 197 + with sentry_sdk.start_transaction(name="test"): + _ = list( + client.chat_completion( + [{"role": "user", "content": "Hello!"}], + stream=True, + ) + ) - if send_default_pii and include_prompts: - expected_data["gen_ai.request.messages"] = ( - '[{"role": "user", "content": "Hello!"}]' - ) - expected_data["gen_ai.response.text"] = "the mocked model response" + (transaction,) = events + + span = None + for sp in transaction["spans"]: + if sp["op"].startswith("gen_ai"): + assert span is None, "there is exactly one gen_ai span" + span = sp + else: + # there should be no other spans, just the gen_ai span + # and optionally some http.client spans from talking to the hf api + assert sp["op"] == "http.client" + + assert span is not None + + assert span["op"] == "gen_ai.chat" + assert span["description"] == "chat test-model" + assert span["origin"] == "auto.ai.huggingface_hub" + + expected_data = { + "gen_ai.operation.name": "chat", + "gen_ai.request.model": "test-model", + "gen_ai.response.finish_reasons": "stop", + "gen_ai.response.model": "test-model-123", + "gen_ai.response.streaming": True, + "thread.id": mock.ANY, + "thread.name": mock.ANY, + } + # usage is not available in older versions of the library + if HF_VERSION and HF_VERSION >= (0, 26, 0): + expected_data["gen_ai.usage.input_tokens"] = 183 + expected_data["gen_ai.usage.output_tokens"] = 14 + expected_data["gen_ai.usage.total_tokens"] = 197 + + if send_default_pii and include_prompts: + expected_data["gen_ai.request.messages"] = ( + '[{"role": "user", "content": "Hello!"}]' + ) + expected_data["gen_ai.response.text"] = "the mocked model response" - if not send_default_pii or not include_prompts: - assert "gen_ai.request.messages" not in expected_data - assert "gen_ai.response.text" not in expected_data + if not send_default_pii or not include_prompts: + assert "gen_ai.request.messages" not in expected_data + assert "gen_ai.response.text" not in expected_data - assert span["data"] == expected_data + assert span["data"] == expected_data @pytest.mark.httpx_mock(assert_all_requests_were_expected=False) @@ -752,50 +765,56 @@ def test_chat_completion_api_error( sentry_init(traces_sample_rate=1.0) events = capture_events() - client = InferenceClient(model="test-model") - - with sentry_sdk.start_transaction(name="test"): - with pytest.raises(HfHubHTTPError): - client.chat_completion( - messages=[{"role": "user", "content": "Hello!"}], - ) - - ( - error, - transaction, - ) = events - - assert error["exception"]["values"][0]["mechanism"]["type"] == "huggingface_hub" - assert not error["exception"]["values"][0]["mechanism"]["handled"] - - span = None - for sp in transaction["spans"]: - if sp["op"].startswith("gen_ai"): - assert span is None, "there is exactly one gen_ai span" - span = sp - else: - # there should be no other spans, just the gen_ai span - # and optionally some http.client spans from talking to the hf api - assert sp["op"] == "http.client" - - assert span is not None - - assert span["op"] == "gen_ai.chat" - assert span["description"] == "chat test-model" - assert span["origin"] == "auto.ai.huggingface_hub" - assert span.get("tags", {}).get("status") == "error" - - assert ( - error["contexts"]["trace"]["trace_id"] - == transaction["contexts"]["trace"]["trace_id"] - ) - expected_data = { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "test-model", - "thread.id": mock.ANY, - "thread.name": mock.ANY, - } - assert span["data"] == expected_data + def get_provider(*args, **kwargs): + return HFInferenceConversational() + + with mock.patch( + "huggingface_hub.inference._client.get_provider_helper", get_provider + ): + client = InferenceClient(model="test-model") + + with sentry_sdk.start_transaction(name="test"): + with pytest.raises(HfHubHTTPError): + client.chat_completion( + messages=[{"role": "user", "content": "Hello!"}], + ) + + ( + error, + transaction, + ) = events + + assert error["exception"]["values"][0]["mechanism"]["type"] == "huggingface_hub" + assert not error["exception"]["values"][0]["mechanism"]["handled"] + + span = None + for sp in transaction["spans"]: + if sp["op"].startswith("gen_ai"): + assert span is None, "there is exactly one gen_ai span" + span = sp + else: + # there should be no other spans, just the gen_ai span + # and optionally some http.client spans from talking to the hf api + assert sp["op"] == "http.client" + + assert span is not None + + assert span["op"] == "gen_ai.chat" + assert span["description"] == "chat test-model" + assert span["origin"] == "auto.ai.huggingface_hub" + assert span.get("tags", {}).get("status") == "error" + + assert ( + error["contexts"]["trace"]["trace_id"] + == transaction["contexts"]["trace"]["trace_id"] + ) + expected_data = { + "gen_ai.operation.name": "chat", + "gen_ai.request.model": "test-model", + "thread.id": mock.ANY, + "thread.name": mock.ANY, + } + assert span["data"] == expected_data @pytest.mark.httpx_mock(assert_all_requests_were_expected=False) @@ -804,31 +823,37 @@ def test_span_status_error(sentry_init, capture_events, mock_hf_api_with_errors) sentry_init(traces_sample_rate=1.0) events = capture_events() - client = InferenceClient(model="test-model") + def get_provider(*args, **kwargs): + return HFInferenceConversational() - with sentry_sdk.start_transaction(name="test"): - with pytest.raises(HfHubHTTPError): - client.chat_completion( - messages=[{"role": "user", "content": "Hello!"}], - ) + with mock.patch( + "huggingface_hub.inference._client.get_provider_helper", get_provider + ): + client = InferenceClient(model="test-model") - (error, transaction) = events - assert error["level"] == "error" + with sentry_sdk.start_transaction(name="test"): + with pytest.raises(HfHubHTTPError): + client.chat_completion( + messages=[{"role": "user", "content": "Hello!"}], + ) - span = None - for sp in transaction["spans"]: - if sp["op"].startswith("gen_ai"): - assert span is None, "there is exactly one gen_ai span" - span = sp - else: - # there should be no other spans, just the gen_ai span - # and optionally some http.client spans from talking to the hf api - assert sp["op"] == "http.client" + (error, transaction) = events + assert error["level"] == "error" - assert span is not None - assert span["tags"]["status"] == "error" + span = None + for sp in transaction["spans"]: + if sp["op"].startswith("gen_ai"): + assert span is None, "there is exactly one gen_ai span" + span = sp + else: + # there should be no other spans, just the gen_ai span + # and optionally some http.client spans from talking to the hf api + assert sp["op"] == "http.client" + + assert span is not None + assert span["tags"]["status"] == "error" - assert transaction["contexts"]["trace"]["status"] == "error" + assert transaction["contexts"]["trace"]["status"] == "error" @pytest.mark.httpx_mock(assert_all_requests_were_expected=False) @@ -849,75 +874,81 @@ def test_chat_completion_with_tools( ) events = capture_events() - client = InferenceClient(model="test-model") - - tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current weather", - "parameters": { - "type": "object", - "properties": {"location": {"type": "string"}}, - "required": ["location"], + def get_provider(*args, **kwargs): + return HFInferenceConversational() + + with mock.patch( + "huggingface_hub.inference._client.get_provider_helper", get_provider + ): + client = InferenceClient(model="test-model") + + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current weather", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, }, - }, - } - ] - - with sentry_sdk.start_transaction(name="test"): - client.chat_completion( - messages=[{"role": "user", "content": "What is the weather in Paris?"}], - tools=tools, - tool_choice="auto", - ) + } + ] - (transaction,) = events - - span = None - for sp in transaction["spans"]: - if sp["op"].startswith("gen_ai"): - assert span is None, "there is exactly one gen_ai span" - span = sp - else: - # there should be no other spans, just the gen_ai span - # and optionally some http.client spans from talking to the hf api - assert sp["op"] == "http.client" - - assert span is not None - - assert span["op"] == "gen_ai.chat" - assert span["description"] == "chat test-model" - assert span["origin"] == "auto.ai.huggingface_hub" + with sentry_sdk.start_transaction(name="test"): + client.chat_completion( + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=tools, + tool_choice="auto", + ) - expected_data = { - "gen_ai.operation.name": "chat", - "gen_ai.request.available_tools": '[{"type": "function", "function": {"name": "get_weather", "description": "Get current weather", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}}}]', - "gen_ai.request.model": "test-model", - "gen_ai.response.finish_reasons": "tool_calls", - "gen_ai.response.model": "test-model-123", - "gen_ai.usage.input_tokens": 10, - "gen_ai.usage.output_tokens": 8, - "gen_ai.usage.total_tokens": 18, - "thread.id": mock.ANY, - "thread.name": mock.ANY, - } + (transaction,) = events + + span = None + for sp in transaction["spans"]: + if sp["op"].startswith("gen_ai"): + assert span is None, "there is exactly one gen_ai span" + span = sp + else: + # there should be no other spans, just the gen_ai span + # and optionally some http.client spans from talking to the hf api + assert sp["op"] == "http.client" + + assert span is not None + + assert span["op"] == "gen_ai.chat" + assert span["description"] == "chat test-model" + assert span["origin"] == "auto.ai.huggingface_hub" + + expected_data = { + "gen_ai.operation.name": "chat", + "gen_ai.request.available_tools": '[{"type": "function", "function": {"name": "get_weather", "description": "Get current weather", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}}}]', + "gen_ai.request.model": "test-model", + "gen_ai.response.finish_reasons": "tool_calls", + "gen_ai.response.model": "test-model-123", + "gen_ai.usage.input_tokens": 10, + "gen_ai.usage.output_tokens": 8, + "gen_ai.usage.total_tokens": 18, + "thread.id": mock.ANY, + "thread.name": mock.ANY, + } - if send_default_pii and include_prompts: - expected_data["gen_ai.request.messages"] = ( - '[{"role": "user", "content": "What is the weather in Paris?"}]' - ) - expected_data["gen_ai.response.tool_calls"] = ( - '[{"function": {"arguments": {"location": "Paris"}, "name": "get_weather", "description": "None"}, "id": "call_123", "type": "function"}]' - ) + if send_default_pii and include_prompts: + expected_data["gen_ai.request.messages"] = ( + '[{"role": "user", "content": "What is the weather in Paris?"}]' + ) + expected_data["gen_ai.response.tool_calls"] = ( + '[{"function": {"arguments": {"location": "Paris"}, "name": "get_weather", "description": "None"}, "id": "call_123", "type": "function"}]' + ) - if not send_default_pii or not include_prompts: - assert "gen_ai.request.messages" not in expected_data - assert "gen_ai.response.text" not in expected_data - assert "gen_ai.response.tool_calls" not in expected_data + if not send_default_pii or not include_prompts: + assert "gen_ai.request.messages" not in expected_data + assert "gen_ai.response.text" not in expected_data + assert "gen_ai.response.tool_calls" not in expected_data - assert span["data"] == expected_data + assert span["data"] == expected_data @pytest.mark.httpx_mock(assert_all_requests_were_expected=False) @@ -938,79 +969,87 @@ def test_chat_completion_streaming_with_tools( ) events = capture_events() - client = InferenceClient(model="test-model") - - tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current weather", - "parameters": { - "type": "object", - "properties": {"location": {"type": "string"}}, - "required": ["location"], + def get_provider(*args, **kwargs): + return HFInferenceConversational() + + with mock.patch( + "huggingface_hub.inference._client.get_provider_helper", get_provider + ): + client = InferenceClient(model="test-model") + + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current weather", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, }, - }, - } - ] - - with sentry_sdk.start_transaction(name="test"): - _ = list( - client.chat_completion( - messages=[{"role": "user", "content": "What is the weather in Paris?"}], - stream=True, - tools=tools, - tool_choice="auto", + } + ] + + with sentry_sdk.start_transaction(name="test"): + _ = list( + client.chat_completion( + messages=[ + {"role": "user", "content": "What is the weather in Paris?"} + ], + stream=True, + tools=tools, + tool_choice="auto", + ) ) - ) - - (transaction,) = events - - span = None - for sp in transaction["spans"]: - if sp["op"].startswith("gen_ai"): - assert span is None, "there is exactly one gen_ai span" - span = sp - else: - # there should be no other spans, just the gen_ai span - # and optionally some http.client spans from talking to the hf api - assert sp["op"] == "http.client" - assert span is not None - - assert span["op"] == "gen_ai.chat" - assert span["description"] == "chat test-model" - assert span["origin"] == "auto.ai.huggingface_hub" - - expected_data = { - "gen_ai.operation.name": "chat", - "gen_ai.request.available_tools": '[{"type": "function", "function": {"name": "get_weather", "description": "Get current weather", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}}}]', - "gen_ai.request.model": "test-model", - "gen_ai.response.finish_reasons": "tool_calls", - "gen_ai.response.model": "test-model-123", - "gen_ai.response.streaming": True, - "thread.id": mock.ANY, - "thread.name": mock.ANY, - } + (transaction,) = events + + span = None + for sp in transaction["spans"]: + if sp["op"].startswith("gen_ai"): + assert span is None, "there is exactly one gen_ai span" + span = sp + else: + # there should be no other spans, just the gen_ai span + # and optionally some http.client spans from talking to the hf api + assert sp["op"] == "http.client" + + assert span is not None + + assert span["op"] == "gen_ai.chat" + assert span["description"] == "chat test-model" + assert span["origin"] == "auto.ai.huggingface_hub" + + expected_data = { + "gen_ai.operation.name": "chat", + "gen_ai.request.available_tools": '[{"type": "function", "function": {"name": "get_weather", "description": "Get current weather", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}}}]', + "gen_ai.request.model": "test-model", + "gen_ai.response.finish_reasons": "tool_calls", + "gen_ai.response.model": "test-model-123", + "gen_ai.response.streaming": True, + "thread.id": mock.ANY, + "thread.name": mock.ANY, + } - if HF_VERSION and HF_VERSION >= (0, 26, 0): - expected_data["gen_ai.usage.input_tokens"] = 183 - expected_data["gen_ai.usage.output_tokens"] = 14 - expected_data["gen_ai.usage.total_tokens"] = 197 + if HF_VERSION and HF_VERSION >= (0, 26, 0): + expected_data["gen_ai.usage.input_tokens"] = 183 + expected_data["gen_ai.usage.output_tokens"] = 14 + expected_data["gen_ai.usage.total_tokens"] = 197 - if send_default_pii and include_prompts: - expected_data["gen_ai.request.messages"] = ( - '[{"role": "user", "content": "What is the weather in Paris?"}]' - ) - expected_data["gen_ai.response.text"] = "response with tool calls follows" - expected_data["gen_ai.response.tool_calls"] = ( - '[{"function": {"arguments": {"location": "Paris"}, "name": "get_weather"}, "id": "call_123", "type": "function", "index": "None"}]' - ) + if send_default_pii and include_prompts: + expected_data["gen_ai.request.messages"] = ( + '[{"role": "user", "content": "What is the weather in Paris?"}]' + ) + expected_data["gen_ai.response.text"] = "response with tool calls follows" + expected_data["gen_ai.response.tool_calls"] = ( + '[{"function": {"arguments": {"location": "Paris"}, "name": "get_weather"}, "id": "call_123", "type": "function", "index": "None"}]' + ) - if not send_default_pii or not include_prompts: - assert "gen_ai.request.messages" not in expected_data - assert "gen_ai.response.text" not in expected_data - assert "gen_ai.response.tool_calls" not in expected_data + if not send_default_pii or not include_prompts: + assert "gen_ai.request.messages" not in expected_data + assert "gen_ai.response.text" not in expected_data + assert "gen_ai.response.tool_calls" not in expected_data - assert span["data"] == expected_data + assert span["data"] == expected_data From 5fdfe01ce946c739925b53ac5a4561c42fa55e2e Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 22 Oct 2025 11:00:15 +0200 Subject: [PATCH 2/7] better use of context managers --- .../huggingface_hub/test_huggingface_hub.py | 472 +++++++++--------- 1 file changed, 236 insertions(+), 236 deletions(-) diff --git a/tests/integrations/huggingface_hub/test_huggingface_hub.py b/tests/integrations/huggingface_hub/test_huggingface_hub.py index e5b08e6383..d8e3fb33e5 100644 --- a/tests/integrations/huggingface_hub/test_huggingface_hub.py +++ b/tests/integrations/huggingface_hub/test_huggingface_hub.py @@ -617,64 +617,64 @@ def test_chat_completion( ) events = capture_events() + client = InferenceClient(model="test-model") + def get_provider(*args, **kwargs): return HFInferenceConversational() with mock.patch( "huggingface_hub.inference._client.get_provider_helper", get_provider ): - client = InferenceClient(model="test-model") - with sentry_sdk.start_transaction(name="test"): client.chat_completion( messages=[{"role": "user", "content": "Hello!"}], stream=False, ) - (transaction,) = events - - span = None - for sp in transaction["spans"]: - if sp["op"].startswith("gen_ai"): - assert span is None, "there is exactly one gen_ai span" - span = sp - else: - # there should be no other spans, just the gen_ai span - # and optionally some http.client spans from talking to the hf api - assert sp["op"] == "http.client" - - assert span is not None - - assert span["op"] == "gen_ai.chat" - assert span["description"] == "chat test-model" - assert span["origin"] == "auto.ai.huggingface_hub" - - expected_data = { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "test-model", - "gen_ai.response.finish_reasons": "stop", - "gen_ai.response.model": "test-model-123", - "gen_ai.response.streaming": False, - "gen_ai.usage.input_tokens": 10, - "gen_ai.usage.output_tokens": 8, - "gen_ai.usage.total_tokens": 18, - "thread.id": mock.ANY, - "thread.name": mock.ANY, - } - - if send_default_pii and include_prompts: - expected_data["gen_ai.request.messages"] = ( - '[{"role": "user", "content": "Hello!"}]' - ) - expected_data["gen_ai.response.text"] = ( - "[mocked] Hello! How can I help you today?" - ) + (transaction,) = events - if not send_default_pii or not include_prompts: - assert "gen_ai.request.messages" not in expected_data - assert "gen_ai.response.text" not in expected_data + span = None + for sp in transaction["spans"]: + if sp["op"].startswith("gen_ai"): + assert span is None, "there is exactly one gen_ai span" + span = sp + else: + # there should be no other spans, just the gen_ai span + # and optionally some http.client spans from talking to the hf api + assert sp["op"] == "http.client" - assert span["data"] == expected_data + assert span is not None + + assert span["op"] == "gen_ai.chat" + assert span["description"] == "chat test-model" + assert span["origin"] == "auto.ai.huggingface_hub" + + expected_data = { + "gen_ai.operation.name": "chat", + "gen_ai.request.model": "test-model", + "gen_ai.response.finish_reasons": "stop", + "gen_ai.response.model": "test-model-123", + "gen_ai.response.streaming": False, + "gen_ai.usage.input_tokens": 10, + "gen_ai.usage.output_tokens": 8, + "gen_ai.usage.total_tokens": 18, + "thread.id": mock.ANY, + "thread.name": mock.ANY, + } + + if send_default_pii and include_prompts: + expected_data["gen_ai.request.messages"] = ( + '[{"role": "user", "content": "Hello!"}]' + ) + expected_data["gen_ai.response.text"] = ( + "[mocked] Hello! How can I help you today?" + ) + + if not send_default_pii or not include_prompts: + assert "gen_ai.request.messages" not in expected_data + assert "gen_ai.response.text" not in expected_data + + assert span["data"] == expected_data @pytest.mark.httpx_mock(assert_all_requests_were_expected=False) @@ -695,14 +695,14 @@ def test_chat_completion_streaming( ) events = capture_events() + client = InferenceClient(model="test-model") + def get_provider(*args, **kwargs): return HFInferenceConversational() with mock.patch( "huggingface_hub.inference._client.get_provider_helper", get_provider ): - client = InferenceClient(model="test-model") - with sentry_sdk.start_transaction(name="test"): _ = list( client.chat_completion( @@ -711,50 +711,50 @@ def get_provider(*args, **kwargs): ) ) - (transaction,) = events - - span = None - for sp in transaction["spans"]: - if sp["op"].startswith("gen_ai"): - assert span is None, "there is exactly one gen_ai span" - span = sp - else: - # there should be no other spans, just the gen_ai span - # and optionally some http.client spans from talking to the hf api - assert sp["op"] == "http.client" - - assert span is not None - - assert span["op"] == "gen_ai.chat" - assert span["description"] == "chat test-model" - assert span["origin"] == "auto.ai.huggingface_hub" - - expected_data = { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "test-model", - "gen_ai.response.finish_reasons": "stop", - "gen_ai.response.model": "test-model-123", - "gen_ai.response.streaming": True, - "thread.id": mock.ANY, - "thread.name": mock.ANY, - } - # usage is not available in older versions of the library - if HF_VERSION and HF_VERSION >= (0, 26, 0): - expected_data["gen_ai.usage.input_tokens"] = 183 - expected_data["gen_ai.usage.output_tokens"] = 14 - expected_data["gen_ai.usage.total_tokens"] = 197 - - if send_default_pii and include_prompts: - expected_data["gen_ai.request.messages"] = ( - '[{"role": "user", "content": "Hello!"}]' - ) - expected_data["gen_ai.response.text"] = "the mocked model response" + (transaction,) = events + + span = None + for sp in transaction["spans"]: + if sp["op"].startswith("gen_ai"): + assert span is None, "there is exactly one gen_ai span" + span = sp + else: + # there should be no other spans, just the gen_ai span + # and optionally some http.client spans from talking to the hf api + assert sp["op"] == "http.client" - if not send_default_pii or not include_prompts: - assert "gen_ai.request.messages" not in expected_data - assert "gen_ai.response.text" not in expected_data + assert span is not None - assert span["data"] == expected_data + assert span["op"] == "gen_ai.chat" + assert span["description"] == "chat test-model" + assert span["origin"] == "auto.ai.huggingface_hub" + + expected_data = { + "gen_ai.operation.name": "chat", + "gen_ai.request.model": "test-model", + "gen_ai.response.finish_reasons": "stop", + "gen_ai.response.model": "test-model-123", + "gen_ai.response.streaming": True, + "thread.id": mock.ANY, + "thread.name": mock.ANY, + } + # usage is not available in older versions of the library + if HF_VERSION and HF_VERSION >= (0, 26, 0): + expected_data["gen_ai.usage.input_tokens"] = 183 + expected_data["gen_ai.usage.output_tokens"] = 14 + expected_data["gen_ai.usage.total_tokens"] = 197 + + if send_default_pii and include_prompts: + expected_data["gen_ai.request.messages"] = ( + '[{"role": "user", "content": "Hello!"}]' + ) + expected_data["gen_ai.response.text"] = "the mocked model response" + + if not send_default_pii or not include_prompts: + assert "gen_ai.request.messages" not in expected_data + assert "gen_ai.response.text" not in expected_data + + assert span["data"] == expected_data @pytest.mark.httpx_mock(assert_all_requests_were_expected=False) @@ -765,56 +765,56 @@ def test_chat_completion_api_error( sentry_init(traces_sample_rate=1.0) events = capture_events() + client = InferenceClient(model="test-model") + def get_provider(*args, **kwargs): return HFInferenceConversational() with mock.patch( "huggingface_hub.inference._client.get_provider_helper", get_provider ): - client = InferenceClient(model="test-model") - with sentry_sdk.start_transaction(name="test"): with pytest.raises(HfHubHTTPError): client.chat_completion( messages=[{"role": "user", "content": "Hello!"}], ) - ( - error, - transaction, - ) = events - - assert error["exception"]["values"][0]["mechanism"]["type"] == "huggingface_hub" - assert not error["exception"]["values"][0]["mechanism"]["handled"] - - span = None - for sp in transaction["spans"]: - if sp["op"].startswith("gen_ai"): - assert span is None, "there is exactly one gen_ai span" - span = sp - else: - # there should be no other spans, just the gen_ai span - # and optionally some http.client spans from talking to the hf api - assert sp["op"] == "http.client" - - assert span is not None - - assert span["op"] == "gen_ai.chat" - assert span["description"] == "chat test-model" - assert span["origin"] == "auto.ai.huggingface_hub" - assert span.get("tags", {}).get("status") == "error" - - assert ( - error["contexts"]["trace"]["trace_id"] - == transaction["contexts"]["trace"]["trace_id"] - ) - expected_data = { - "gen_ai.operation.name": "chat", - "gen_ai.request.model": "test-model", - "thread.id": mock.ANY, - "thread.name": mock.ANY, - } - assert span["data"] == expected_data + ( + error, + transaction, + ) = events + + assert error["exception"]["values"][0]["mechanism"]["type"] == "huggingface_hub" + assert not error["exception"]["values"][0]["mechanism"]["handled"] + + span = None + for sp in transaction["spans"]: + if sp["op"].startswith("gen_ai"): + assert span is None, "there is exactly one gen_ai span" + span = sp + else: + # there should be no other spans, just the gen_ai span + # and optionally some http.client spans from talking to the hf api + assert sp["op"] == "http.client" + + assert span is not None + + assert span["op"] == "gen_ai.chat" + assert span["description"] == "chat test-model" + assert span["origin"] == "auto.ai.huggingface_hub" + assert span.get("tags", {}).get("status") == "error" + + assert ( + error["contexts"]["trace"]["trace_id"] + == transaction["contexts"]["trace"]["trace_id"] + ) + expected_data = { + "gen_ai.operation.name": "chat", + "gen_ai.request.model": "test-model", + "thread.id": mock.ANY, + "thread.name": mock.ANY, + } + assert span["data"] == expected_data @pytest.mark.httpx_mock(assert_all_requests_were_expected=False) @@ -823,37 +823,37 @@ def test_span_status_error(sentry_init, capture_events, mock_hf_api_with_errors) sentry_init(traces_sample_rate=1.0) events = capture_events() + client = InferenceClient(model="test-model") + def get_provider(*args, **kwargs): return HFInferenceConversational() with mock.patch( "huggingface_hub.inference._client.get_provider_helper", get_provider ): - client = InferenceClient(model="test-model") - with sentry_sdk.start_transaction(name="test"): with pytest.raises(HfHubHTTPError): client.chat_completion( messages=[{"role": "user", "content": "Hello!"}], ) - (error, transaction) = events - assert error["level"] == "error" + (error, transaction) = events + assert error["level"] == "error" - span = None - for sp in transaction["spans"]: - if sp["op"].startswith("gen_ai"): - assert span is None, "there is exactly one gen_ai span" - span = sp - else: - # there should be no other spans, just the gen_ai span - # and optionally some http.client spans from talking to the hf api - assert sp["op"] == "http.client" + span = None + for sp in transaction["spans"]: + if sp["op"].startswith("gen_ai"): + assert span is None, "there is exactly one gen_ai span" + span = sp + else: + # there should be no other spans, just the gen_ai span + # and optionally some http.client spans from talking to the hf api + assert sp["op"] == "http.client" - assert span is not None - assert span["tags"]["status"] == "error" + assert span is not None + assert span["tags"]["status"] == "error" - assert transaction["contexts"]["trace"]["status"] == "error" + assert transaction["contexts"]["trace"]["status"] == "error" @pytest.mark.httpx_mock(assert_all_requests_were_expected=False) @@ -874,14 +874,14 @@ def test_chat_completion_with_tools( ) events = capture_events() + client = InferenceClient(model="test-model") + def get_provider(*args, **kwargs): return HFInferenceConversational() with mock.patch( "huggingface_hub.inference._client.get_provider_helper", get_provider ): - client = InferenceClient(model="test-model") - tools = [ { "type": "function", @@ -904,51 +904,51 @@ def get_provider(*args, **kwargs): tool_choice="auto", ) - (transaction,) = events - - span = None - for sp in transaction["spans"]: - if sp["op"].startswith("gen_ai"): - assert span is None, "there is exactly one gen_ai span" - span = sp - else: - # there should be no other spans, just the gen_ai span - # and optionally some http.client spans from talking to the hf api - assert sp["op"] == "http.client" - - assert span is not None - - assert span["op"] == "gen_ai.chat" - assert span["description"] == "chat test-model" - assert span["origin"] == "auto.ai.huggingface_hub" - - expected_data = { - "gen_ai.operation.name": "chat", - "gen_ai.request.available_tools": '[{"type": "function", "function": {"name": "get_weather", "description": "Get current weather", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}}}]', - "gen_ai.request.model": "test-model", - "gen_ai.response.finish_reasons": "tool_calls", - "gen_ai.response.model": "test-model-123", - "gen_ai.usage.input_tokens": 10, - "gen_ai.usage.output_tokens": 8, - "gen_ai.usage.total_tokens": 18, - "thread.id": mock.ANY, - "thread.name": mock.ANY, - } - - if send_default_pii and include_prompts: - expected_data["gen_ai.request.messages"] = ( - '[{"role": "user", "content": "What is the weather in Paris?"}]' - ) - expected_data["gen_ai.response.tool_calls"] = ( - '[{"function": {"arguments": {"location": "Paris"}, "name": "get_weather", "description": "None"}, "id": "call_123", "type": "function"}]' - ) + (transaction,) = events + + span = None + for sp in transaction["spans"]: + if sp["op"].startswith("gen_ai"): + assert span is None, "there is exactly one gen_ai span" + span = sp + else: + # there should be no other spans, just the gen_ai span + # and optionally some http.client spans from talking to the hf api + assert sp["op"] == "http.client" - if not send_default_pii or not include_prompts: - assert "gen_ai.request.messages" not in expected_data - assert "gen_ai.response.text" not in expected_data - assert "gen_ai.response.tool_calls" not in expected_data + assert span is not None - assert span["data"] == expected_data + assert span["op"] == "gen_ai.chat" + assert span["description"] == "chat test-model" + assert span["origin"] == "auto.ai.huggingface_hub" + + expected_data = { + "gen_ai.operation.name": "chat", + "gen_ai.request.available_tools": '[{"type": "function", "function": {"name": "get_weather", "description": "Get current weather", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}}}]', + "gen_ai.request.model": "test-model", + "gen_ai.response.finish_reasons": "tool_calls", + "gen_ai.response.model": "test-model-123", + "gen_ai.usage.input_tokens": 10, + "gen_ai.usage.output_tokens": 8, + "gen_ai.usage.total_tokens": 18, + "thread.id": mock.ANY, + "thread.name": mock.ANY, + } + + if send_default_pii and include_prompts: + expected_data["gen_ai.request.messages"] = ( + '[{"role": "user", "content": "What is the weather in Paris?"}]' + ) + expected_data["gen_ai.response.tool_calls"] = ( + '[{"function": {"arguments": {"location": "Paris"}, "name": "get_weather", "description": "None"}, "id": "call_123", "type": "function"}]' + ) + + if not send_default_pii or not include_prompts: + assert "gen_ai.request.messages" not in expected_data + assert "gen_ai.response.text" not in expected_data + assert "gen_ai.response.tool_calls" not in expected_data + + assert span["data"] == expected_data @pytest.mark.httpx_mock(assert_all_requests_were_expected=False) @@ -969,14 +969,14 @@ def test_chat_completion_streaming_with_tools( ) events = capture_events() + client = InferenceClient(model="test-model") + def get_provider(*args, **kwargs): return HFInferenceConversational() with mock.patch( "huggingface_hub.inference._client.get_provider_helper", get_provider ): - client = InferenceClient(model="test-model") - tools = [ { "type": "function", @@ -1004,52 +1004,52 @@ def get_provider(*args, **kwargs): ) ) - (transaction,) = events - - span = None - for sp in transaction["spans"]: - if sp["op"].startswith("gen_ai"): - assert span is None, "there is exactly one gen_ai span" - span = sp - else: - # there should be no other spans, just the gen_ai span - # and optionally some http.client spans from talking to the hf api - assert sp["op"] == "http.client" - - assert span is not None - - assert span["op"] == "gen_ai.chat" - assert span["description"] == "chat test-model" - assert span["origin"] == "auto.ai.huggingface_hub" - - expected_data = { - "gen_ai.operation.name": "chat", - "gen_ai.request.available_tools": '[{"type": "function", "function": {"name": "get_weather", "description": "Get current weather", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}}}]', - "gen_ai.request.model": "test-model", - "gen_ai.response.finish_reasons": "tool_calls", - "gen_ai.response.model": "test-model-123", - "gen_ai.response.streaming": True, - "thread.id": mock.ANY, - "thread.name": mock.ANY, - } - - if HF_VERSION and HF_VERSION >= (0, 26, 0): - expected_data["gen_ai.usage.input_tokens"] = 183 - expected_data["gen_ai.usage.output_tokens"] = 14 - expected_data["gen_ai.usage.total_tokens"] = 197 - - if send_default_pii and include_prompts: - expected_data["gen_ai.request.messages"] = ( - '[{"role": "user", "content": "What is the weather in Paris?"}]' - ) - expected_data["gen_ai.response.text"] = "response with tool calls follows" - expected_data["gen_ai.response.tool_calls"] = ( - '[{"function": {"arguments": {"location": "Paris"}, "name": "get_weather"}, "id": "call_123", "type": "function", "index": "None"}]' - ) + (transaction,) = events + + span = None + for sp in transaction["spans"]: + if sp["op"].startswith("gen_ai"): + assert span is None, "there is exactly one gen_ai span" + span = sp + else: + # there should be no other spans, just the gen_ai span + # and optionally some http.client spans from talking to the hf api + assert sp["op"] == "http.client" + + assert span is not None + + assert span["op"] == "gen_ai.chat" + assert span["description"] == "chat test-model" + assert span["origin"] == "auto.ai.huggingface_hub" + + expected_data = { + "gen_ai.operation.name": "chat", + "gen_ai.request.available_tools": '[{"type": "function", "function": {"name": "get_weather", "description": "Get current weather", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}}}]', + "gen_ai.request.model": "test-model", + "gen_ai.response.finish_reasons": "tool_calls", + "gen_ai.response.model": "test-model-123", + "gen_ai.response.streaming": True, + "thread.id": mock.ANY, + "thread.name": mock.ANY, + } + + if HF_VERSION and HF_VERSION >= (0, 26, 0): + expected_data["gen_ai.usage.input_tokens"] = 183 + expected_data["gen_ai.usage.output_tokens"] = 14 + expected_data["gen_ai.usage.total_tokens"] = 197 - if not send_default_pii or not include_prompts: - assert "gen_ai.request.messages" not in expected_data - assert "gen_ai.response.text" not in expected_data - assert "gen_ai.response.tool_calls" not in expected_data + if send_default_pii and include_prompts: + expected_data["gen_ai.request.messages"] = ( + '[{"role": "user", "content": "What is the weather in Paris?"}]' + ) + expected_data["gen_ai.response.text"] = "response with tool calls follows" + expected_data["gen_ai.response.tool_calls"] = ( + '[{"function": {"arguments": {"location": "Paris"}, "name": "get_weather"}, "id": "call_123", "type": "function", "index": "None"}]' + ) - assert span["data"] == expected_data + if not send_default_pii or not include_prompts: + assert "gen_ai.request.messages" not in expected_data + assert "gen_ai.response.text" not in expected_data + assert "gen_ai.response.tool_calls" not in expected_data + + assert span["data"] == expected_data From 29039fe84d09be78fa5143dca7e89b4c7345166d Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 22 Oct 2025 11:16:34 +0200 Subject: [PATCH 3/7] use public api instead of mock --- .../huggingface_hub/test_huggingface_hub.py | 178 +++++++----------- 1 file changed, 70 insertions(+), 108 deletions(-) diff --git a/tests/integrations/huggingface_hub/test_huggingface_hub.py b/tests/integrations/huggingface_hub/test_huggingface_hub.py index d8e3fb33e5..19ff33422a 100644 --- a/tests/integrations/huggingface_hub/test_huggingface_hub.py +++ b/tests/integrations/huggingface_hub/test_huggingface_hub.py @@ -617,19 +617,13 @@ def test_chat_completion( ) events = capture_events() - client = InferenceClient(model="test-model") - - def get_provider(*args, **kwargs): - return HFInferenceConversational() + client = InferenceClient(model="test-model", provider="hf-inference") - with mock.patch( - "huggingface_hub.inference._client.get_provider_helper", get_provider - ): - with sentry_sdk.start_transaction(name="test"): - client.chat_completion( - messages=[{"role": "user", "content": "Hello!"}], - stream=False, - ) + with sentry_sdk.start_transaction(name="test"): + client.chat_completion( + messages=[{"role": "user", "content": "Hello!"}], + stream=False, + ) (transaction,) = events @@ -695,21 +689,15 @@ def test_chat_completion_streaming( ) events = capture_events() - client = InferenceClient(model="test-model") + client = InferenceClient(model="test-model", provider="hf-inference") - def get_provider(*args, **kwargs): - return HFInferenceConversational() - - with mock.patch( - "huggingface_hub.inference._client.get_provider_helper", get_provider - ): - with sentry_sdk.start_transaction(name="test"): - _ = list( - client.chat_completion( - [{"role": "user", "content": "Hello!"}], - stream=True, - ) + with sentry_sdk.start_transaction(name="test"): + _ = list( + client.chat_completion( + [{"role": "user", "content": "Hello!"}], + stream=True, ) + ) (transaction,) = events @@ -765,19 +753,13 @@ def test_chat_completion_api_error( sentry_init(traces_sample_rate=1.0) events = capture_events() - client = InferenceClient(model="test-model") - - def get_provider(*args, **kwargs): - return HFInferenceConversational() + client = InferenceClient(model="test-model", provider="hf-inference") - with mock.patch( - "huggingface_hub.inference._client.get_provider_helper", get_provider - ): - with sentry_sdk.start_transaction(name="test"): - with pytest.raises(HfHubHTTPError): - client.chat_completion( - messages=[{"role": "user", "content": "Hello!"}], - ) + with sentry_sdk.start_transaction(name="test"): + with pytest.raises(HfHubHTTPError): + client.chat_completion( + messages=[{"role": "user", "content": "Hello!"}], + ) ( error, @@ -823,19 +805,13 @@ def test_span_status_error(sentry_init, capture_events, mock_hf_api_with_errors) sentry_init(traces_sample_rate=1.0) events = capture_events() - client = InferenceClient(model="test-model") - - def get_provider(*args, **kwargs): - return HFInferenceConversational() + client = InferenceClient(model="test-model", provider="hf-inference") - with mock.patch( - "huggingface_hub.inference._client.get_provider_helper", get_provider - ): - with sentry_sdk.start_transaction(name="test"): - with pytest.raises(HfHubHTTPError): - client.chat_completion( - messages=[{"role": "user", "content": "Hello!"}], - ) + with sentry_sdk.start_transaction(name="test"): + with pytest.raises(HfHubHTTPError): + client.chat_completion( + messages=[{"role": "user", "content": "Hello!"}], + ) (error, transaction) = events assert error["level"] == "error" @@ -874,35 +850,29 @@ def test_chat_completion_with_tools( ) events = capture_events() - client = InferenceClient(model="test-model") - - def get_provider(*args, **kwargs): - return HFInferenceConversational() - - with mock.patch( - "huggingface_hub.inference._client.get_provider_helper", get_provider - ): - tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current weather", - "parameters": { - "type": "object", - "properties": {"location": {"type": "string"}}, - "required": ["location"], - }, + client = InferenceClient(model="test-model", provider="hf-inference") + + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current weather", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], }, - } - ] + }, + } + ] - with sentry_sdk.start_transaction(name="test"): - client.chat_completion( - messages=[{"role": "user", "content": "What is the weather in Paris?"}], - tools=tools, - tool_choice="auto", - ) + with sentry_sdk.start_transaction(name="test"): + client.chat_completion( + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=tools, + tool_choice="auto", + ) (transaction,) = events @@ -969,40 +939,32 @@ def test_chat_completion_streaming_with_tools( ) events = capture_events() - client = InferenceClient(model="test-model") - - def get_provider(*args, **kwargs): - return HFInferenceConversational() - - with mock.patch( - "huggingface_hub.inference._client.get_provider_helper", get_provider - ): - tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current weather", - "parameters": { - "type": "object", - "properties": {"location": {"type": "string"}}, - "required": ["location"], - }, + client = InferenceClient(model="test-model", provider="hf-inference") + + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current weather", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], }, - } - ] - - with sentry_sdk.start_transaction(name="test"): - _ = list( - client.chat_completion( - messages=[ - {"role": "user", "content": "What is the weather in Paris?"} - ], - stream=True, - tools=tools, - tool_choice="auto", - ) + }, + } + ] + + with sentry_sdk.start_transaction(name="test"): + _ = list( + client.chat_completion( + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + stream=True, + tools=tools, + tool_choice="auto", ) + ) (transaction,) = events From 6a92459de2ee5be5cfbea072d7f478475e88fcfb Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 22 Oct 2025 11:17:17 +0200 Subject: [PATCH 4/7] remove import --- tests/integrations/huggingface_hub/test_huggingface_hub.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integrations/huggingface_hub/test_huggingface_hub.py b/tests/integrations/huggingface_hub/test_huggingface_hub.py index 19ff33422a..0389295313 100644 --- a/tests/integrations/huggingface_hub/test_huggingface_hub.py +++ b/tests/integrations/huggingface_hub/test_huggingface_hub.py @@ -5,7 +5,6 @@ import httpx from huggingface_hub import InferenceClient -from huggingface_hub.inference._providers.hf_inference import HFInferenceConversational import sentry_sdk from sentry_sdk.utils import package_version From 7eaee761911f065ba2c355b5c990c2bec5ef102b Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 22 Oct 2025 13:00:08 +0200 Subject: [PATCH 5/7] version gate option --- .../huggingface_hub/test_huggingface_hub.py | 42 ++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/tests/integrations/huggingface_hub/test_huggingface_hub.py b/tests/integrations/huggingface_hub/test_huggingface_hub.py index 0389295313..12d52f438a 100644 --- a/tests/integrations/huggingface_hub/test_huggingface_hub.py +++ b/tests/integrations/huggingface_hub/test_huggingface_hub.py @@ -616,7 +616,12 @@ def test_chat_completion( ) events = capture_events() - client = InferenceClient(model="test-model", provider="hf-inference") + # The provider parameter was added in 0.28.0 of huggingface_hub + client = ( + InferenceClient(model="test-model", provider="hf-inference") + if HF_VERSION >= (0, 28, 0) + else InferenceClient(model="test-model") + ) with sentry_sdk.start_transaction(name="test"): client.chat_completion( @@ -688,7 +693,12 @@ def test_chat_completion_streaming( ) events = capture_events() - client = InferenceClient(model="test-model", provider="hf-inference") + # The provider parameter was added in 0.28.0 of huggingface_hub + client = ( + InferenceClient(model="test-model", provider="hf-inference") + if HF_VERSION >= (0, 28, 0) + else InferenceClient(model="test-model") + ) with sentry_sdk.start_transaction(name="test"): _ = list( @@ -752,7 +762,12 @@ def test_chat_completion_api_error( sentry_init(traces_sample_rate=1.0) events = capture_events() - client = InferenceClient(model="test-model", provider="hf-inference") + # The provider parameter was added in 0.28.0 of huggingface_hub + client = ( + InferenceClient(model="test-model", provider="hf-inference") + if HF_VERSION >= (0, 28, 0) + else InferenceClient(model="test-model") + ) with sentry_sdk.start_transaction(name="test"): with pytest.raises(HfHubHTTPError): @@ -804,7 +819,12 @@ def test_span_status_error(sentry_init, capture_events, mock_hf_api_with_errors) sentry_init(traces_sample_rate=1.0) events = capture_events() - client = InferenceClient(model="test-model", provider="hf-inference") + # The provider parameter was added in 0.28.0 of huggingface_hub + client = ( + InferenceClient(model="test-model", provider="hf-inference") + if HF_VERSION >= (0, 28, 0) + else InferenceClient(model="test-model") + ) with sentry_sdk.start_transaction(name="test"): with pytest.raises(HfHubHTTPError): @@ -849,7 +869,12 @@ def test_chat_completion_with_tools( ) events = capture_events() - client = InferenceClient(model="test-model", provider="hf-inference") + # The provider parameter was added in 0.28.0 of huggingface_hub + client = ( + InferenceClient(model="test-model", provider="hf-inference") + if HF_VERSION >= (0, 28, 0) + else InferenceClient(model="test-model") + ) tools = [ { @@ -938,7 +963,12 @@ def test_chat_completion_streaming_with_tools( ) events = capture_events() - client = InferenceClient(model="test-model", provider="hf-inference") + # The provider parameter was added in 0.28.0 of huggingface_hub + client = ( + InferenceClient(model="test-model", provider="hf-inference") + if HF_VERSION >= (0, 28, 0) + else InferenceClient(model="test-model") + ) tools = [ { From 391a7abd7343e41a656c68a679c5b3b390e0a5df Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 22 Oct 2025 13:03:27 +0200 Subject: [PATCH 6/7] comment --- .../huggingface_hub/test_huggingface_hub.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/integrations/huggingface_hub/test_huggingface_hub.py b/tests/integrations/huggingface_hub/test_huggingface_hub.py index 12d52f438a..1cb70085b9 100644 --- a/tests/integrations/huggingface_hub/test_huggingface_hub.py +++ b/tests/integrations/huggingface_hub/test_huggingface_hub.py @@ -616,7 +616,7 @@ def test_chat_completion( ) events = capture_events() - # The provider parameter was added in 0.28.0 of huggingface_hub + # The provider parameter was added in version 0.28.0 of huggingface_hub client = ( InferenceClient(model="test-model", provider="hf-inference") if HF_VERSION >= (0, 28, 0) @@ -693,7 +693,7 @@ def test_chat_completion_streaming( ) events = capture_events() - # The provider parameter was added in 0.28.0 of huggingface_hub + # The provider parameter was added in version 0.28.0 of huggingface_hub client = ( InferenceClient(model="test-model", provider="hf-inference") if HF_VERSION >= (0, 28, 0) @@ -762,7 +762,7 @@ def test_chat_completion_api_error( sentry_init(traces_sample_rate=1.0) events = capture_events() - # The provider parameter was added in 0.28.0 of huggingface_hub + # The provider parameter was added in version 0.28.0 of huggingface_hub client = ( InferenceClient(model="test-model", provider="hf-inference") if HF_VERSION >= (0, 28, 0) @@ -819,7 +819,7 @@ def test_span_status_error(sentry_init, capture_events, mock_hf_api_with_errors) sentry_init(traces_sample_rate=1.0) events = capture_events() - # The provider parameter was added in 0.28.0 of huggingface_hub + # The provider parameter was added in version 0.28.0 of huggingface_hub client = ( InferenceClient(model="test-model", provider="hf-inference") if HF_VERSION >= (0, 28, 0) @@ -869,7 +869,7 @@ def test_chat_completion_with_tools( ) events = capture_events() - # The provider parameter was added in 0.28.0 of huggingface_hub + # The provider parameter was added in version 0.28.0 of huggingface_hub client = ( InferenceClient(model="test-model", provider="hf-inference") if HF_VERSION >= (0, 28, 0) @@ -963,7 +963,7 @@ def test_chat_completion_streaming_with_tools( ) events = capture_events() - # The provider parameter was added in 0.28.0 of huggingface_hub + # The provider parameter was added in version 0.28.0 of huggingface_hub client = ( InferenceClient(model="test-model", provider="hf-inference") if HF_VERSION >= (0, 28, 0) From 0000dd2e1c2681748bbcd18fb89d840a2a0c3e9d Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 22 Oct 2025 13:44:24 +0200 Subject: [PATCH 7/7] dedupe --- .../huggingface_hub/test_huggingface_hub.py | 51 ++++++------------- 1 file changed, 15 insertions(+), 36 deletions(-) diff --git a/tests/integrations/huggingface_hub/test_huggingface_hub.py b/tests/integrations/huggingface_hub/test_huggingface_hub.py index 1cb70085b9..ffeb6acbb5 100644 --- a/tests/integrations/huggingface_hub/test_huggingface_hub.py +++ b/tests/integrations/huggingface_hub/test_huggingface_hub.py @@ -34,6 +34,15 @@ ) +def get_hf_provider_inference_client(): + # The provider parameter was added in version 0.28.0 of huggingface_hub + return ( + InferenceClient(model="test-model", provider="hf-inference") + if HF_VERSION >= (0, 28, 0) + else InferenceClient(model="test-model") + ) + + def _add_mock_response( httpx_mock, rsps, method, url, json=None, status=200, body=None, headers=None ): @@ -616,12 +625,7 @@ def test_chat_completion( ) events = capture_events() - # The provider parameter was added in version 0.28.0 of huggingface_hub - client = ( - InferenceClient(model="test-model", provider="hf-inference") - if HF_VERSION >= (0, 28, 0) - else InferenceClient(model="test-model") - ) + client = get_hf_provider_inference_client() with sentry_sdk.start_transaction(name="test"): client.chat_completion( @@ -693,12 +697,7 @@ def test_chat_completion_streaming( ) events = capture_events() - # The provider parameter was added in version 0.28.0 of huggingface_hub - client = ( - InferenceClient(model="test-model", provider="hf-inference") - if HF_VERSION >= (0, 28, 0) - else InferenceClient(model="test-model") - ) + client = get_hf_provider_inference_client() with sentry_sdk.start_transaction(name="test"): _ = list( @@ -762,12 +761,7 @@ def test_chat_completion_api_error( sentry_init(traces_sample_rate=1.0) events = capture_events() - # The provider parameter was added in version 0.28.0 of huggingface_hub - client = ( - InferenceClient(model="test-model", provider="hf-inference") - if HF_VERSION >= (0, 28, 0) - else InferenceClient(model="test-model") - ) + client = get_hf_provider_inference_client() with sentry_sdk.start_transaction(name="test"): with pytest.raises(HfHubHTTPError): @@ -819,12 +813,7 @@ def test_span_status_error(sentry_init, capture_events, mock_hf_api_with_errors) sentry_init(traces_sample_rate=1.0) events = capture_events() - # The provider parameter was added in version 0.28.0 of huggingface_hub - client = ( - InferenceClient(model="test-model", provider="hf-inference") - if HF_VERSION >= (0, 28, 0) - else InferenceClient(model="test-model") - ) + client = get_hf_provider_inference_client() with sentry_sdk.start_transaction(name="test"): with pytest.raises(HfHubHTTPError): @@ -869,12 +858,7 @@ def test_chat_completion_with_tools( ) events = capture_events() - # The provider parameter was added in version 0.28.0 of huggingface_hub - client = ( - InferenceClient(model="test-model", provider="hf-inference") - if HF_VERSION >= (0, 28, 0) - else InferenceClient(model="test-model") - ) + client = get_hf_provider_inference_client() tools = [ { @@ -963,12 +947,7 @@ def test_chat_completion_streaming_with_tools( ) events = capture_events() - # The provider parameter was added in version 0.28.0 of huggingface_hub - client = ( - InferenceClient(model="test-model", provider="hf-inference") - if HF_VERSION >= (0, 28, 0) - else InferenceClient(model="test-model") - ) + client = get_hf_provider_inference_client() tools = [ {