diff --git a/.github/workflows/genai-tools.yml b/.github/workflows/genai-tools.yml index 333bee3e11..7cf8cb5f7f 100644 --- a/.github/workflows/genai-tools.yml +++ b/.github/workflows/genai-tools.yml @@ -90,28 +90,31 @@ jobs: working-directory: ${{ env.OV_INSTALL_DIR }} - name: Test native pytorch model run: | - git clone --depth 1 https://huggingface.co/katuni4ka/tiny-random-qwen - python ./tools/llm_bench/benchmark.py -m tiny-random-qwen -d cpu -n 1 -f pt -ic 20 - rm -rf tiny-random-qwen - env: - GIT_LFS_SKIP_SMUDGE: 0 - - name: Test tiny-random-baichuan2 Optimum Intel + huggingface-cli download katuni4ka/tiny-random-qwen --local-dir ./tiny-random-qwen + python ./tools/llm_bench/benchmark.py -m ./tiny-random-qwen -d cpu -n 1 -f pt -ic 20 + rm -rf ./tiny-random-qwen + - name: Test katuni4ka/tiny-random-baichuan2 Optimum Intel run: | - optimum-cli export openvino --model katuni4ka/tiny-random-baichuan2 --trust-remote-code --weight-format fp16 ./ov_models/tiny-random-baichuan2/pytorch/dldt/FP16 - python ./tools/llm_bench/benchmark.py -m ./ov_models/tiny-random-baichuan2/pytorch/dldt/FP16/ -d cpu -n 1 --optimum -ic 10 - rm -rf ./ov_models/tiny-random-baichuan2 - - name: Test OpenVINO/LCM_Dreamshaper_v7-int8-ov Optimum Intel + optimum-cli export openvino --model katuni4ka/tiny-random-baichuan2 --trust-remote-code --weight-format fp16 ./ov_models/tiny-random-baichuan2 + python ./tools/llm_bench/benchmark.py -m ./ov_models/tiny-random-baichuan2/ -d cpu -n 1 --optimum -ic 10 + rm -rf ./ov_models/ + - name: Create prompt file for for image generation run: | - huggingface-cli download OpenVINO/LCM_Dreamshaper_v7-int8-ov --local-dir ov_models/lcm_dreamshaper_v7 - python ./tools/llm_bench/benchmark.py -m ./ov_models/lcm_dreamshaper_v7/ -pf ./tools/llm_bench/prompts/stable-diffusion.jsonl -d cpu -n 1 --optimum --num_steps 4 - - name: Test OpenVINO/LCM_Dreamshaper_v7-int8-ov with GenAI + prompt="side profile centered painted portrait, Gandhi rolling a blunt, Gloomhaven, matte painting concept art, art nouveau, 8K HD Resolution, beautifully background" + json_template='{steps: $steps, width: $width, height: $height, guidance_scale: $guidance_scale, prompt: $prompt}' + jq -n -c --arg steps "30" --arg width "64" --arg height "128" --arg guidance_scale "1.0" --arg prompt "$prompt" "$json_template" > ./image_generation.jsonl + jq -n -c --arg steps "4" --arg width "64" --arg height "32" --arg guidance_scale "7.0" --arg prompt "$prompt" "$json_template" >> ./image_generation.jsonl + - name: Test echarlaix/tiny-random-latent-consistency Optimum Intel run: | - python ./tools/llm_bench/benchmark.py -m ./ov_models/lcm_dreamshaper_v7/ -pf ./tools/llm_bench/prompts/stable-diffusion.jsonl -d cpu -n 1 --num_steps 4 - - name: Test OpenVINO/LCM_Dreamshaper_v7-int8-ov with GenAI and LoRA + optimum-cli export openvino --model echarlaix/tiny-random-latent-consistency --trust-remote-code --weight-format fp16 ./ov_models/tiny-random-latent-consistency + python ./tools/llm_bench/benchmark.py -m ./ov_models/tiny-random-latent-consistency/ -pf ./image_generation.jsonl -d cpu -n 1 --optimum --num_steps 4 + - name: Test echarlaix/tiny-random-latent-consistency with GenAI + run: python ./tools/llm_bench/benchmark.py -m ./ov_models/tiny-random-latent-consistency/ -pf ./image_generation.jsonl -d cpu -n 1 --num_steps 4 + - name: Test echarlaix/tiny-random-latent-consistency with GenAI and LoRA run: | - wget -O ./ov_models/soulcard.safetensors https://civitai.com/api/download/models/72591 - python ./tools/llm_bench/benchmark.py -m ./ov_models/lcm_dreamshaper_v7/ -pf ./tools/llm_bench/prompts/stable-diffusion.jsonl -d cpu -n 1 --lora ./ov_models/soulcard.safetensors --lora_alphas 0.7 --num_steps 4 - rm -rf ./ov_models/lcm_dreamshaper_v7/ + huggingface-cli download katuni4ka/tiny-random-latent-consistency-lora --local-dir ./lora + python ./tools/llm_bench/benchmark.py -m ./ov_models/tiny-random-latent-consistency/ -pf ./image_generation.jsonl -d cpu -n 1 --num_steps 4 --lora ./lora/tiny-random-latent-consistency-lora.safetensors --lora_alphas 0.7 + rm -rf ./ov_models/ ./lora ./image_generation.jsonl - name: Test TinyLlama-1.1B-Chat-v1.0 in Speculative Decoding via GenAI run: | optimum-cli export openvino --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 --trust-remote-code --weight-format fp16 ov_models/TinyLlama-1.1B-Chat-v1.0/FP16 @@ -119,7 +122,7 @@ jobs: python ./tools/llm_bench/benchmark.py -m ./ov_models/TinyLlama-1.1B-Chat-v1.0/FP16/ --draft_model ./ov_models/TinyLlama-1.1B-Chat-v1.0/INT8/ -p "Why is the Sun yellow?" -d cpu --draft_device cpu -n 1 --assistant_confidence_threshold 0.4 -ic 20 python ./tools/llm_bench/benchmark.py -m ./ov_models/TinyLlama-1.1B-Chat-v1.0/FP16/ --draft_model ./ov_models/TinyLlama-1.1B-Chat-v1.0/INT8/ -p "Why is the Sun yellow?" -d cpu --draft_device cpu -n 1 --num_assistant_tokens 5 -ic 20 rm -rf ov_models/TinyLlama-1.1B-Chat-v1.0 - - name: Test whisper-tiny via GenAI + - name: Test openai/whisper-tiny via Optimum run: | GIT_LFS_SKIP_SMUDGE=1 git clone --depth 1 --branch main --single-branch https://huggingface.co/datasets/facebook/multilingual_librispeech cd multilingual_librispeech @@ -129,15 +132,19 @@ jobs: cd .. optimum-cli export openvino --trust-remote-code --model openai/whisper-tiny ./ov_models/whisper-tiny python ./tools/llm_bench/benchmark.py -m ./ov_models/whisper-tiny --media multilingual_librispeech/data/mls_polish/train/audio/3283_1447_000/3283_1447_000000.flac -d cpu -n 1 --optimum + - name: Test openai/whisper-tiny via GenAI + run: | python ./tools/llm_bench/benchmark.py -m ./ov_models/whisper-tiny --media multilingual_librispeech/data/mls_polish/train/audio/3283_1447_000/3283_1447_000000.flac -d cpu -n 1 rm -rf ./ov_models/whisper-tiny rm -rf multilingual_librispeech - - name: Text InternVL2-1B via GenAI + - name: Test katuni4ka/tiny-random-llava via Optimum + run: | + optimum-cli export openvino --model katuni4ka/tiny-random-llava ./ov_models/tiny-random-llava --task image-text-to-text --trust-remote-code + python ./tools/llm_bench/benchmark.py -m ./ov_models/tiny-random-llava --media https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/d5fbbd1a-d484-415c-88cb-9986625b7b11 --prompt "What is unusual on this image?" -ic 20 --optimum + - name: Test katuni4ka/tiny-random-llava via GenAI run: | - optimum-cli export openvino --model OpenGVLab/InternVL2-1B ./ov_models/internvl2-1B --task image-text-to-text --trust-remote-code - python ./tools/llm_bench/benchmark.py -m ./ov_models/internvl2-1B --media https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/d5fbbd1a-d484-415c-88cb-9986625b7b11 --prompt "What is unusual on this image?" -ic 20 - python ./tools/llm_bench/benchmark.py -m ./ov_models/internvl2-1B --media https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/d5fbbd1a-d484-415c-88cb-9986625b7b11 --prompt "What is unusual on this image?" -ic 20 --optimum - rm -rf ./ov_models/internvl2-1B + python ./tools/llm_bench/benchmark.py -m ./ov_models/tiny-random-llava --media https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/d5fbbd1a-d484-415c-88cb-9986625b7b11 --prompt "What is unusual on this image?" -ic 20 + rm -rf ./ov_models wwb: name: 'WWB tests' diff --git a/src/cpp/src/image_generation/flux_pipeline.hpp b/src/cpp/src/image_generation/flux_pipeline.hpp index e74cd441ce..458caf001b 100644 --- a/src/cpp/src/image_generation/flux_pipeline.hpp +++ b/src/cpp/src/image_generation/flux_pipeline.hpp @@ -252,7 +252,7 @@ class FluxPipeline : public DiffusionPipeline { m_vae->compile(device, properties); m_transformer->compile(device, properties); } - + void compute_hidden_states(const std::string& positive_prompt, const ImageGenerationConfig& generation_config) override { // encode_prompt std::string prompt_2_str = generation_config.prompt_2 != std::nullopt ? *generation_config.prompt_2 : positive_prompt; diff --git a/tools/who_what_benchmark/tests/test_cli_image.py b/tools/who_what_benchmark/tests/test_cli_image.py index f3792a60fb..1ad8236058 100644 --- a/tools/who_what_benchmark/tests/test_cli_image.py +++ b/tools/who_what_benchmark/tests/test_cli_image.py @@ -12,7 +12,9 @@ logger = logging.getLogger(__name__) MODEL_CACHE = tempfile.mkdtemp() -OV_IMAGE_MODELS = ["OpenVINO/stable-diffusion-v1-5-int8-ov"] +OV_IMAGE_MODELS = ["echarlaix/tiny-random-stable-diffusion-xl", + "yujiepan/stable-diffusion-3-tiny-random", + "katuni4ka/tiny-random-flux"] def run_wwb(args): @@ -25,9 +27,7 @@ def run_wwb(args): def setup_module(): for model_id in OV_IMAGE_MODELS: MODEL_PATH = os.path.join(MODEL_CACHE, model_id.replace("/", "--")) - subprocess.run(["huggingface-cli", "download", - model_id, "--local-dir", - MODEL_PATH], capture_output=True, text=True) + subprocess.run(["optimum-cli", "export", "openvino", "--model", model_id, MODEL_PATH], capture_output=True, text=True) def teardown_module(): @@ -103,6 +103,9 @@ def test_image_model_types(model_id, model_type, backend): ])), ) def test_image_model_genai(model_id, model_type): + if ("flux" in model_id or "stable-diffusion-3" in model_id) and model_type != "text-to-image": + pytest.skip(reason="FLUX or SD3 are supported as text to image only") + with tempfile.TemporaryDirectory() as temp_dir: GT_FILE = os.path.join(temp_dir, "gt.csv") MODEL_PATH = os.path.join(MODEL_CACHE, model_id.replace("/", "--"))