From b4a5d42230eb97792904579420c96ec86c78acd0 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Fri, 17 Jul 2026 23:20:14 -0700 Subject: [PATCH 1/2] test(integration): skip Azure key-auth-disabled target tests Key-based (local) auth is administratively disabled in our Azure tenant (Entra/AAD only), so a batch of integration target tests fail with HTTP 403 AuthenticationTypeDisabled. These endpoints already have passing Entra-auth coverage (test_entra_auth_targets.py), so the key-auth variants are redundant functional coverage. Apply a static @pytest.mark.skip with a shared per-file reason constant to the 35 confirmed AuthenticationTypeDisabled nodes across four files, plus two adjacent score tests that fail on the same missing-key root cause: test_targets_and_secrets.py (21), test_openai_responses_gpt5.py (5, module pytestmark), test_target_filters.py (4), test_notebooks_targets.py (5), score/test_azure_content_filter_integration.py (2). Out-of-scope failures (missing env, quota, realtime wss, AML, DeepSeek, Phi4, non-Azure providers) are left untouched. Test-only change; no production code. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 127c6e28-016f-4a16-99aa-43780f8a3c22 --- .../test_azure_content_filter_integration.py | 7 ++ .../targets/test_notebooks_targets.py | 31 ++++++- .../targets/test_openai_responses_gpt5.py | 7 ++ .../targets/test_target_filters.py | 9 ++ .../targets/test_targets_and_secrets.py | 92 ++++++++++++++++--- 5 files changed, 127 insertions(+), 19 deletions(-) diff --git a/tests/integration/score/test_azure_content_filter_integration.py b/tests/integration/score/test_azure_content_filter_integration.py index 9599325081..8c8034210f 100644 --- a/tests/integration/score/test_azure_content_filter_integration.py +++ b/tests/integration/score/test_azure_content_filter_integration.py @@ -12,6 +12,11 @@ from pyrit.memory import CentralMemory, MemoryInterface from pyrit.score import AzureContentFilterScorer +_AZURE_KEY_AUTH_DISABLED_REASON = ( + "Azure key-based (local) auth is disabled in our tenant; " + "covered by the Entra-auth tests (test_entra_auth_targets.py)." +) + @pytest.fixture def memory() -> Generator[MemoryInterface, None, None]: @@ -80,6 +85,7 @@ async def test_azure_content_filter_scorer_long_text_chunking_integration(memory @pytest.mark.run_only_if_all_tests +@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON) async def test_azure_content_filter_scorer_image_with_api_key(memory) -> None: """ Integration test for Azure Content Filter Scorer image scoring with explicit API key auth. @@ -108,6 +114,7 @@ async def test_azure_content_filter_scorer_image_with_api_key(memory) -> None: @pytest.mark.run_only_if_all_tests +@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON) async def test_azure_content_filter_scorer_text_with_api_key(memory) -> None: """ Integration test for Azure Content Filter Scorer text scoring with explicit API key auth. diff --git a/tests/integration/targets/test_notebooks_targets.py b/tests/integration/targets/test_notebooks_targets.py index 4cd50c7dbc..b677403baa 100644 --- a/tests/integration/targets/test_notebooks_targets.py +++ b/tests/integration/targets/test_notebooks_targets.py @@ -19,11 +19,34 @@ "10_3_websocket_copilot_target.ipynb", # WebSocket Copilot target requires manual pasting tokens ] - -@pytest.mark.parametrize( - "file_name", - [file for file in os.listdir(nb_directory_path) if file.endswith(".ipynb") and file not in skipped_files], +_AZURE_KEY_AUTH_DISABLED_REASON = ( + "Azure key-based (local) auth is disabled in our tenant; " + "covered by the Entra-auth tests (test_entra_auth_targets.py)." ) + +# Notebooks whose targets use Azure key-based (local) auth, which is disabled in our tenant. +_azure_key_auth_notebooks = { + "10_http_target.ipynb", + "5_openai_tts_target.ipynb", + "6_custom_targets.ipynb", + "9_rate_limiting.ipynb", + "round_robin_target.ipynb", +} + + +def _notebook_params(): + params = [] + for file in os.listdir(nb_directory_path): + if not file.endswith(".ipynb") or file in skipped_files: + continue + if file in _azure_key_auth_notebooks: + params.append(pytest.param(file, marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON))) + else: + params.append(file) + return params + + +@pytest.mark.parametrize("file_name", _notebook_params()) def test_execute_notebooks(file_name): nb_path = pathlib.Path(nb_directory_path, file_name).resolve() with open(nb_path, encoding="utf-8") as f: diff --git a/tests/integration/targets/test_openai_responses_gpt5.py b/tests/integration/targets/test_openai_responses_gpt5.py index 7a7b184b07..ba2fc6ebd8 100644 --- a/tests/integration/targets/test_openai_responses_gpt5.py +++ b/tests/integration/targets/test_openai_responses_gpt5.py @@ -12,6 +12,13 @@ from pyrit.models import MessagePiece from pyrit.prompt_target import OpenAIResponseTarget +_AZURE_KEY_AUTH_DISABLED_REASON = ( + "Azure key-based (local) auth is disabled in our tenant; " + "covered by the Entra-auth tests (test_entra_auth_targets.py)." +) + +pytestmark = pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON) + @pytest.fixture() def gpt5_args(): diff --git a/tests/integration/targets/test_target_filters.py b/tests/integration/targets/test_target_filters.py index 25d77be313..03c10054ec 100644 --- a/tests/integration/targets/test_target_filters.py +++ b/tests/integration/targets/test_target_filters.py @@ -13,6 +13,11 @@ OpenAIVideoTarget, ) +_AZURE_KEY_AUTH_DISABLED_REASON = ( + "Azure key-based (local) auth is disabled in our tenant; " + "covered by the Entra-auth tests (test_entra_auth_targets.py)." +) + @pytest.mark.parametrize( ("endpoint", "api_key", "model_name"), @@ -24,6 +29,7 @@ ), ], ) +@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON) async def test_azure_content_filters(sqlite_instance, endpoint, api_key, model_name): args = { "endpoint": os.getenv(endpoint), @@ -62,6 +68,7 @@ async def test_azure_content_filters(sqlite_instance, endpoint, api_key, model_n ), ], ) +@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON) async def test_azure_content_filters_response_api(sqlite_instance, endpoint, api_key, model_name): endpoint_val = os.getenv(endpoint) api_key_val = os.getenv(api_key) @@ -95,6 +102,7 @@ async def test_azure_content_filters_response_api(sqlite_instance, endpoint, api ("endpoint", "api_key", "model_name"), [("OPENAI_IMAGE_STRICT_FILTER_ENDPOINT", "OPENAI_IMAGE_STRICT_FILTER_KEY", "OPENAI_IMAGE_STRICT_FILTER_MODEL")], ) +@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON) async def test_image_input_filters(sqlite_instance, endpoint, api_key, model_name): target = OpenAIImageTarget( endpoint=os.getenv(endpoint), api_key=os.getenv(api_key), model_name=os.getenv(model_name) @@ -119,6 +127,7 @@ async def test_image_input_filters(sqlite_instance, endpoint, api_key, model_nam ("endpoint", "api_key", "model_name"), [("AZURE_OPENAI_VIDEO_ENDPOINT", "AZURE_OPENAI_VIDEO_KEY", "AZURE_OPENAI_VIDEO_MODEL")], ) +@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON) async def test_video_input_filters(sqlite_instance, endpoint, api_key, model_name): target = OpenAIVideoTarget( endpoint=os.getenv(endpoint), diff --git a/tests/integration/targets/test_targets_and_secrets.py b/tests/integration/targets/test_targets_and_secrets.py index a546664161..5f51a92d65 100644 --- a/tests/integration/targets/test_targets_and_secrets.py +++ b/tests/integration/targets/test_targets_and_secrets.py @@ -23,6 +23,11 @@ RealtimeTarget, ) +_AZURE_KEY_AUTH_DISABLED_REASON = ( + "Azure key-based (local) auth is disabled in our tenant; " + "covered by the Entra-auth tests (test_entra_auth_targets.py)." +) + def _get_required_env_var(env_var_name: str) -> str: """ @@ -94,42 +99,77 @@ async def _assert_can_send_video_prompt(target): @pytest.mark.parametrize( ("endpoint", "api_key", "model_name", "supports_seed"), [ - ("OPENAI_CHAT_ENDPOINT", "OPENAI_CHAT_KEY", "OPENAI_CHAT_MODEL", True), + pytest.param( + "OPENAI_CHAT_ENDPOINT", + "OPENAI_CHAT_KEY", + "OPENAI_CHAT_MODEL", + True, + marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON), + ), ("PLATFORM_OPENAI_CHAT_ENDPOINT", "PLATFORM_OPENAI_CHAT_KEY", "PLATFORM_OPENAI_CHAT_MODEL", True), - ("AZURE_OPENAI_GPT4O_ENDPOINT", "AZURE_OPENAI_GPT4O_KEY", "AZURE_OPENAI_GPT4O_MODEL", True), - ( + pytest.param( + "AZURE_OPENAI_GPT4O_ENDPOINT", + "AZURE_OPENAI_GPT4O_KEY", + "AZURE_OPENAI_GPT4O_MODEL", + True, + marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON), + ), + pytest.param( "AZURE_OPENAI_INTEGRATION_TEST_ENDPOINT", "AZURE_OPENAI_INTEGRATION_TEST_KEY", "AZURE_OPENAI_INTEGRATION_TEST_MODEL", True, + marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON), ), - ( + pytest.param( "AZURE_OPENAI_GPT4O_UNSAFE_CHAT_ENDPOINT", "AZURE_OPENAI_GPT4O_UNSAFE_CHAT_KEY", "AZURE_OPENAI_GPT4O_UNSAFE_CHAT_MODEL", True, + marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON), ), - ( + pytest.param( "AZURE_OPENAI_GPT4O_UNSAFE_CHAT_ENDPOINT2", "AZURE_OPENAI_GPT4O_UNSAFE_CHAT_KEY2", "AZURE_OPENAI_GPT4O_UNSAFE_CHAT_MODEL2", True, + marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON), ), - ("AZURE_OPENAI_GPT3_5_CHAT_ENDPOINT", "AZURE_OPENAI_GPT3_5_CHAT_KEY", "AZURE_OPENAI_GPT3_5_CHAT_MODEL", True), - ("AZURE_OPENAI_GPT4_CHAT_ENDPOINT", "AZURE_OPENAI_GPT4_CHAT_KEY", "AZURE_OPENAI_GPT4_CHAT_MODEL", True), - ("AZURE_OPENAI_GPTV_CHAT_ENDPOINT", "AZURE_OPENAI_GPTV_CHAT_KEY", "AZURE_OPENAI_GPTV_CHAT_MODEL", True), - ( + pytest.param( + "AZURE_OPENAI_GPT3_5_CHAT_ENDPOINT", + "AZURE_OPENAI_GPT3_5_CHAT_KEY", + "AZURE_OPENAI_GPT3_5_CHAT_MODEL", + True, + marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON), + ), + pytest.param( + "AZURE_OPENAI_GPT4_CHAT_ENDPOINT", + "AZURE_OPENAI_GPT4_CHAT_KEY", + "AZURE_OPENAI_GPT4_CHAT_MODEL", + True, + marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON), + ), + pytest.param( + "AZURE_OPENAI_GPTV_CHAT_ENDPOINT", + "AZURE_OPENAI_GPTV_CHAT_KEY", + "AZURE_OPENAI_GPTV_CHAT_MODEL", + True, + marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON), + ), + pytest.param( "AZURE_OPENAI_GPT5_COMPLETIONS_ENDPOINT", "AZURE_OPENAI_GPT5_COMPLETIONS_KEY", "AZURE_OPENAI_GPT5_COMPLETIONS_MODEL", True, + marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON), ), ("AZURE_FOUNDRY_DEEPSEEK_ENDPOINT", "AZURE_FOUNDRY_DEEPSEEK_KEY", "AZURE_FOUNDRY_DEEPSEEK_MODEL", True), - ( + pytest.param( "AZURE_FOUNDRY_MISTRAL_LARGE_ENDPOINT", "AZURE_FOUNDRY_MISTRAL_LARGE_KEY", "AZURE_FOUNDRY_MISTRAL_LARGE_MODEL", False, + marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON), ), ("AZURE_FOUNDRY_PHI4_ENDPOINT", "AZURE_CHAT_PHI4_KEY", "AZURE_CHAT_PHI4_MODEL", True), ("GOOGLE_GEMINI_ENDPOINT", "GOOGLE_GEMINI_API_KEY", "GOOGLE_GEMINI_MODEL", False), @@ -166,16 +206,23 @@ async def test_connect_required_openai_text_targets(sqlite_instance, endpoint, a "PLATFORM_OPENAI_RESPONSES_KEY", "PLATFORM_OPENAI_RESPONSES_MODEL", ), - ("AZURE_OPENAI_RESPONSES_ENDPOINT", "AZURE_OPENAI_RESPONSES_KEY", "AZURE_OPENAI_RESPONSES_MODEL"), - ( + pytest.param( + "AZURE_OPENAI_RESPONSES_ENDPOINT", + "AZURE_OPENAI_RESPONSES_KEY", + "AZURE_OPENAI_RESPONSES_MODEL", + marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON), + ), + pytest.param( "AZURE_OPENAI_GPT41_RESPONSES_ENDPOINT", "AZURE_OPENAI_GPT41_RESPONSES_KEY", "AZURE_OPENAI_GPT41_RESPONSES_MODEL", + marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON), ), - ( + pytest.param( "AZURE_OPENAI_GPT5_RESPONSES_ENDPOINT", "AZURE_OPENAI_GPT5_KEY", "AZURE_OPENAI_GPT5_MODEL", + marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON), ), ("AWS_ENDPOINT", "AWS_KEY", "AWS_RESPONSES_MODEL"), ], @@ -294,8 +341,18 @@ async def test_connect_openai_completion(sqlite_instance): @pytest.mark.parametrize( ("endpoint", "api_key", "model_name"), [ - ("OPENAI_IMAGE_ENDPOINT1", "OPENAI_IMAGE_API_KEY1", "OPENAI_IMAGE_MODEL1"), # gpt-image-1.5 - ("OPENAI_IMAGE_ENDPOINT2", "OPENAI_IMAGE_API_KEY2", "OPENAI_IMAGE_MODEL2"), # gpt-image-1 + pytest.param( + "OPENAI_IMAGE_ENDPOINT1", + "OPENAI_IMAGE_API_KEY1", + "OPENAI_IMAGE_MODEL1", + marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON), + ), # gpt-image-1.5 + pytest.param( + "OPENAI_IMAGE_ENDPOINT2", + "OPENAI_IMAGE_API_KEY2", + "OPENAI_IMAGE_MODEL2", + marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON), + ), # gpt-image-1 ("PLATFORM_OPENAI_IMAGE_ENDPOINT", "PLATFORM_OPENAI_IMAGE_KEY", "PLATFORM_OPENAI_IMAGE_MODEL"), # gpt-image-1.5 ], ) @@ -332,6 +389,7 @@ async def test_connect_image(sqlite_instance, endpoint, api_key, model_name): @pytest.mark.run_only_if_all_tests +@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON) async def test_image_editing_single_image_api_key(sqlite_instance): """ Test image editing with a single image input using API key authentication. @@ -380,6 +438,7 @@ async def test_image_editing_single_image_api_key(sqlite_instance): @pytest.mark.run_only_if_all_tests +@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON) async def test_image_editing_multiple_images_api_key(sqlite_instance): """ Test image editing with multiple image inputs using API key authentication. @@ -440,6 +499,7 @@ async def test_image_editing_multiple_images_api_key(sqlite_instance): ("OPENAI_TTS_ENDPOINT2", "OPENAI_TTS_KEY2", "OPENAI_TTS_MODEL2"), ], ) +@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON) async def test_connect_tts(sqlite_instance, endpoint, api_key, model_name): endpoint_value = _get_required_env_var(endpoint) api_key_value = _get_required_env_var(api_key) @@ -464,6 +524,7 @@ async def test_connect_tts(sqlite_instance, endpoint, api_key, model_name): # "PLATFORM_OPENAI_VIDEO_MODEL"), ], ) +@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON) async def test_connect_video(sqlite_instance, endpoint, api_key, model_name): """Test OpenAIVideoTarget with video API.""" endpoint_value = _get_required_env_var(endpoint) @@ -482,6 +543,7 @@ async def test_connect_video(sqlite_instance, endpoint, api_key, model_name): @pytest.mark.run_only_if_all_tests +@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON) async def test_video_multiple_prompts_create_separate_files(sqlite_instance): """ Test that sending multiple prompts to video API using PromptSendingAttack From 3ba7f23c0151dd5095457ce3bff682509419d9ca Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Sat, 18 Jul 2026 17:36:42 -0700 Subject: [PATCH 2/2] test(integration): skip Azure key-auth-disabled embedding param Extends the plain-skip gating to the Azure OPENAI_EMBEDDING param of test_openai_embedding_with_api_key, for parity with the content-filter *_with_api_key skips. PLATFORM_OPENAI_EMBEDDING stays active/ungated. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 127c6e28-016f-4a16-99aa-43780f8a3c22 --- .../integration/embeddings/test_openai_embedding.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/integration/embeddings/test_openai_embedding.py b/tests/integration/embeddings/test_openai_embedding.py index 26195c8539..58e234b0a2 100644 --- a/tests/integration/embeddings/test_openai_embedding.py +++ b/tests/integration/embeddings/test_openai_embedding.py @@ -8,12 +8,22 @@ from pyrit.auth import get_azure_openai_auth from pyrit.embedding import OpenAITextEmbedding +_AZURE_KEY_AUTH_DISABLED_REASON = ( + "Azure key-based (local) auth is disabled in our tenant; " + "covered by the Entra-auth tests (test_entra_auth_targets.py)." +) + @pytest.mark.run_only_if_all_tests @pytest.mark.parametrize( "endpoint_env,key_env,model_env", [ - ("OPENAI_EMBEDDING_ENDPOINT", "OPENAI_EMBEDDING_KEY", "OPENAI_EMBEDDING_MODEL"), + pytest.param( + "OPENAI_EMBEDDING_ENDPOINT", + "OPENAI_EMBEDDING_KEY", + "OPENAI_EMBEDDING_MODEL", + marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON), + ), ("PLATFORM_OPENAI_EMBEDDING_ENDPOINT", "PLATFORM_OPENAI_EMBEDDING_KEY", "PLATFORM_OPENAI_EMBEDDING_MODEL"), ], )