From dcad32e5031253a3fdd9db2867ca3e8fe7cc86ac Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Thu, 9 Jan 2025 02:30:06 -0800 Subject: [PATCH] Python: Temporarily xFail some google ai integration tests to unblock (#10136) ### Motivation and Context This pull request includes updates to the test cases in the `python/tests/integration/completions/test_chat_completion_with_function_calling.py` and `python/tests/integration/completions/test_chat_completion_with_image_input_text_output.py` files. The changes primarily address the handling of Google AI API issues by adding markers to the tests. ### Description * [`python/tests/integration/completions/test_chat_completion_with_function_calling.py`](diffhunk://#diff-530ae449ac82bcd42280dcf440a135e38acc53a5362940fdbb64edfdc4300abcL506-R512): Added `xfail` markers to multiple test cases to handle temporary failures due to Google AI API issues related to function calling not being enabled for models/gemini-1.5-flash. [[1]](diffhunk://#diff-530ae449ac82bcd42280dcf440a135e38acc53a5362940fdbb64edfdc4300abcL506-R512) [[2]](diffhunk://#diff-530ae449ac82bcd42280dcf440a135e38acc53a5362940fdbb64edfdc4300abcL526-R538) [[3]](diffhunk://#diff-530ae449ac82bcd42280dcf440a135e38acc53a5362940fdbb64edfdc4300abcL572-R590) * [`python/tests/integration/completions/test_chat_completion_with_image_input_text_output.py`](diffhunk://#diff-9c1b24d0f91106612c3b55b1f63122811cf4d3cb6dbc6908e50b8a3804bd42ecL186-R192): Added an `xfail` marker to handle temporary failure due to Google AI API issues related to image input modality not being enabled for models/gemini-1.5-flash. ### Contribution Checklist - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone :smile: --- ...t_chat_completion_with_function_calling.py | 24 ++++++++++++++++--- ...completion_with_image_input_text_output.py | 8 ++++++- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/python/tests/integration/completions/test_chat_completion_with_function_calling.py b/python/tests/integration/completions/test_chat_completion_with_function_calling.py index 6760eb5a1b02..db47eaccf7f5 100644 --- a/python/tests/integration/completions/test_chat_completion_with_function_calling.py +++ b/python/tests/integration/completions/test_chat_completion_with_function_calling.py @@ -503,7 +503,13 @@ class FunctionChoiceTestTypes(str, Enum): ] ], {"test_type": FunctionChoiceTestTypes.AUTO}, - marks=pytest.mark.skipif(not google_ai_setup, reason="Google AI Environment Variables not set"), + marks=[ + pytest.mark.skipif(not google_ai_setup, reason="Google AI Environment Variables not set"), + pytest.mark.xfail( + reason="Temporarily failing due to Google AI API issue on streaming only: " + "400 Function calling is not enabled for models/gemini-1.5-flash" + ), + ], id="google_ai_tool_call_auto", ), pytest.param( @@ -523,7 +529,13 @@ class FunctionChoiceTestTypes(str, Enum): ] ], {"test_type": FunctionChoiceTestTypes.NON_AUTO}, - marks=pytest.mark.skipif(not google_ai_setup, reason="Google AI Environment Variables not set"), + marks=[ + pytest.mark.skipif(not google_ai_setup, reason="Google AI Environment Variables not set"), + pytest.mark.xfail( + reason="Temporarily failing due to Google AI API issue on streaming only: " + "400 Function calling is not enabled for models/gemini-1.5-flash" + ), + ], id="google_ai_tool_call_non_auto", ), pytest.param( @@ -569,7 +581,13 @@ class FunctionChoiceTestTypes(str, Enum): ] ], {"test_type": FunctionChoiceTestTypes.AUTO}, - marks=pytest.mark.skipif(not google_ai_setup, reason="Google AI Environment Variables not set"), + marks=[ + pytest.mark.skipif(not google_ai_setup, reason="Google AI Environment Variables not set"), + pytest.mark.xfail( + reason="Temporarily failing due to Google AI API issue on streaming only: " + "400 Function calling is not enabled for models/gemini-1.5-flash" + ), + ], id="google_ai_tool_call_auto_complex_return_type", ), # endregion diff --git a/python/tests/integration/completions/test_chat_completion_with_image_input_text_output.py b/python/tests/integration/completions/test_chat_completion_with_image_input_text_output.py index 2eb52e22f1e5..a0cd1b384f5b 100644 --- a/python/tests/integration/completions/test_chat_completion_with_image_input_text_output.py +++ b/python/tests/integration/completions/test_chat_completion_with_image_input_text_output.py @@ -183,7 +183,13 @@ ), ], {}, - marks=pytest.mark.skipif(not google_ai_setup, reason="Google AI Environment Variables not set"), + marks=[ + pytest.mark.skipif(not google_ai_setup, reason="Google AI Environment Variables not set"), + pytest.mark.xfail( + reason="Temporarily failing due to Google AI API issue on streaming only: " + "400 Image input modality is not enabled for models/gemini-1.5-flash" + ), + ], id="google_ai_image_input_file", ), pytest.param(