From 7360155388b02141fbaf24a625dfaa2fc7880951 Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Tue, 10 Sep 2024 14:00:53 -0400 Subject: [PATCH 1/4] add integration tests for HF endpoint --- .../tests/integration_tests/test_standard.py | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 libs/partners/huggingface/tests/integration_tests/test_standard.py diff --git a/libs/partners/huggingface/tests/integration_tests/test_standard.py b/libs/partners/huggingface/tests/integration_tests/test_standard.py new file mode 100644 index 0000000000000..8e07265295adb --- /dev/null +++ b/libs/partners/huggingface/tests/integration_tests/test_standard.py @@ -0,0 +1,78 @@ +"""Standard LangChain interface tests""" + +from typing import Type + +from langchain_core.language_models import BaseChatModel +from langchain_standard_tests.integration_tests import ChatModelIntegrationTests +import pytest + +from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint, HuggingFacePipeline + + +class TestHuggingFaceEndpoint(ChatModelIntegrationTests): + @property + def chat_model_class(self) -> Type[BaseChatModel]: + return ChatHuggingFace + + @property + def chat_model_params(self) -> dict: + return {} + + @pytest.fixture + def model(self) -> BaseChatModel: + llm = HuggingFaceEndpoint( + repo_id="HuggingFaceH4/zephyr-7b-beta", + task="text-generation", + max_new_tokens=512, + do_sample=False, + repetition_penalty=1.03, + ) + return self.chat_model_class(llm=llm) + + @pytest.mark.xfail(reason=("Not implemented")) + def test_stream(self, model: BaseChatModel) -> None: + super().test_stream(model) + + @pytest.mark.xfail(reason=("Not implemented")) + async def test_astream(self, model: BaseChatModel) -> None: + await super().test_astream(model) + + @pytest.mark.xfail(reason=("Not implemented")) + def test_usage_metadata(self, model: BaseChatModel) -> None: + super().test_usage_metadata(model) + + @pytest.mark.xfail(reason=("Not implemented")) + def test_usage_metadata_streaming(self, model: BaseChatModel) -> None: + super().test_usage_metadata_streaming(model) + + @pytest.mark.xfail(reason=("Not implemented")) + def test_stop_sequence(self, model: BaseChatModel) -> None: + super().test_stop_sequence(model) + + @pytest.mark.xfail(reason=("Not implemented")) + def test_tool_calling(self, model: BaseChatModel) -> None: + super().test_tool_calling(model) + + @pytest.mark.xfail(reason=("Not implemented")) + def test_tool_calling_with_no_arguments(self, model: BaseChatModel) -> None: + super().test_tool_calling_with_no_arguments(model) + + @pytest.mark.xfail(reason=("Not implemented")) + def test_bind_runnables_as_tools(self, model: BaseChatModel) -> None: + super().test_bind_runnables_as_tools(model) + + @pytest.mark.xfail(reason=("Not implemented")) + def test_structured_output(self, model: BaseChatModel) -> None: + super().test_structured_output(model) + + @pytest.mark.xfail(reason=("Not implemented")) + def test_structured_output_pydantic_2_v1(self, model: BaseChatModel) -> None: + super().test_structured_output_pydantic_2_v1(model) + + @pytest.mark.xfail(reason=("Not implemented")) + def test_tool_message_histories_list_content(self, model: BaseChatModel) -> None: + super().test_tool_message_histories_list_content(model) + + @pytest.mark.xfail(reason=("Not implemented")) + def test_structured_few_shot_examples(self, model: BaseChatModel) -> None: + super().test_structured_few_shot_examples(model) From c3b81f81fd76075e6cfde563469f234c595ac23a Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Tue, 10 Sep 2024 14:12:49 -0400 Subject: [PATCH 2/4] add api token to workflow secrets --- .github/workflows/_integration_test.yml | 1 + .github/workflows/_release.yml | 1 + .github/workflows/scheduled_test.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/_integration_test.yml b/.github/workflows/_integration_test.yml index 05971e1dfc5f7..40b0ec5344edd 100644 --- a/.github/workflows/_integration_test.yml +++ b/.github/workflows/_integration_test.yml @@ -67,6 +67,7 @@ jobs: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} GOOGLE_SEARCH_API_KEY: ${{ secrets.GOOGLE_SEARCH_API_KEY }} GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }} + HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} EXA_API_KEY: ${{ secrets.EXA_API_KEY }} NOMIC_API_KEY: ${{ secrets.NOMIC_API_KEY }} WATSONX_APIKEY: ${{ secrets.WATSONX_APIKEY }} diff --git a/.github/workflows/_release.yml b/.github/workflows/_release.yml index 68da8c3a7045b..d3efddf67eb17 100644 --- a/.github/workflows/_release.yml +++ b/.github/workflows/_release.yml @@ -273,6 +273,7 @@ jobs: GOOGLE_SEARCH_API_KEY: ${{ secrets.GOOGLE_SEARCH_API_KEY }} GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }} GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} EXA_API_KEY: ${{ secrets.EXA_API_KEY }} NOMIC_API_KEY: ${{ secrets.NOMIC_API_KEY }} WATSONX_APIKEY: ${{ secrets.WATSONX_APIKEY }} diff --git a/.github/workflows/scheduled_test.yml b/.github/workflows/scheduled_test.yml index 61e1b3d25a6ae..b317a48e51535 100644 --- a/.github/workflows/scheduled_test.yml +++ b/.github/workflows/scheduled_test.yml @@ -90,6 +90,7 @@ jobs: AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME }} FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} From 5ad4ebe5fdc0e235be4269d8ebec0874e7402b45 Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Tue, 10 Sep 2024 14:19:02 -0400 Subject: [PATCH 3/4] lint --- .../tests/integration_tests/test_standard.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libs/partners/huggingface/tests/integration_tests/test_standard.py b/libs/partners/huggingface/tests/integration_tests/test_standard.py index 8e07265295adb..f9180b4771141 100644 --- a/libs/partners/huggingface/tests/integration_tests/test_standard.py +++ b/libs/partners/huggingface/tests/integration_tests/test_standard.py @@ -2,11 +2,11 @@ from typing import Type +import pytest from langchain_core.language_models import BaseChatModel from langchain_standard_tests.integration_tests import ChatModelIntegrationTests -import pytest -from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint, HuggingFacePipeline +from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint class TestHuggingFaceEndpoint(ChatModelIntegrationTests): @@ -20,14 +20,14 @@ def chat_model_params(self) -> dict: @pytest.fixture def model(self) -> BaseChatModel: - llm = HuggingFaceEndpoint( + llm = HuggingFaceEndpoint( # type: ignore[call-arg] repo_id="HuggingFaceH4/zephyr-7b-beta", task="text-generation", max_new_tokens=512, do_sample=False, repetition_penalty=1.03, ) - return self.chat_model_class(llm=llm) + return self.chat_model_class(llm=llm) # type: ignore[call-arg] @pytest.mark.xfail(reason=("Not implemented")) def test_stream(self, model: BaseChatModel) -> None: @@ -68,11 +68,11 @@ def test_structured_output(self, model: BaseChatModel) -> None: @pytest.mark.xfail(reason=("Not implemented")) def test_structured_output_pydantic_2_v1(self, model: BaseChatModel) -> None: super().test_structured_output_pydantic_2_v1(model) - + @pytest.mark.xfail(reason=("Not implemented")) def test_tool_message_histories_list_content(self, model: BaseChatModel) -> None: super().test_tool_message_histories_list_content(model) - + @pytest.mark.xfail(reason=("Not implemented")) def test_structured_few_shot_examples(self, model: BaseChatModel) -> None: super().test_structured_few_shot_examples(model) From a01ff11c58480a119273161fe03aa5ec9afeaadc Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Tue, 10 Sep 2024 14:39:52 -0400 Subject: [PATCH 4/4] add tests for embeddings --- .../test_embeddings_standard.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 libs/partners/huggingface/tests/integration_tests/test_embeddings_standard.py diff --git a/libs/partners/huggingface/tests/integration_tests/test_embeddings_standard.py b/libs/partners/huggingface/tests/integration_tests/test_embeddings_standard.py new file mode 100644 index 0000000000000..4f5cefcbdff2c --- /dev/null +++ b/libs/partners/huggingface/tests/integration_tests/test_embeddings_standard.py @@ -0,0 +1,30 @@ +"""Test HuggingFace embeddings.""" + +from typing import Type + +from langchain_standard_tests.integration_tests import EmbeddingsIntegrationTests + +from langchain_huggingface.embeddings import ( + HuggingFaceEmbeddings, + HuggingFaceEndpointEmbeddings, +) + + +class TestHuggingFaceEmbeddings(EmbeddingsIntegrationTests): + @property + def embeddings_class(self) -> Type[HuggingFaceEmbeddings]: + return HuggingFaceEmbeddings + + @property + def embedding_model_params(self) -> dict: + return {"model_name": "sentence-transformers/all-mpnet-base-v2"} + + +class TestHuggingFaceEndpointEmbeddings(EmbeddingsIntegrationTests): + @property + def embeddings_class(self) -> Type[HuggingFaceEndpointEmbeddings]: + return HuggingFaceEndpointEmbeddings + + @property + def embedding_model_params(self) -> dict: + return {"model": "sentence-transformers/all-mpnet-base-v2"}