add performance statistics for image generation (openvinotoolkit#1405)

tickets: CVS-157338

---------

Co-authored-by: Ilya Lavrenov <ilya.lavrenov@intel.com>

.github/workflows/lcm_dreamshaper_cpp.yml

@@ -51,7 +51,7 @@ jobs:
         run: |
           source ${{ env.OV_INSTALL_DIR }}/setupvars.sh
           cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ${{ env.build_dir }}
-          cmake --build ${{ env.build_dir }} --config Release --target text2image image2image inpainting heterogeneous_stable_diffusion lora_text2image py_openvino_genai --parallel
+          cmake --build ${{ env.build_dir }} --config Release --target text2image image2image inpainting heterogeneous_stable_diffusion lora_text2image benchmark_image_gen py_openvino_genai --parallel
 
       - name: Create virtual environment
         run: python3 -m venv openvino_lcm_cpp
@@ -92,6 +92,21 @@ jobs:
           source ./ov/setupvars.sh
           ${{ env.build_dir }}/samples/cpp/image_generation/inpainting ./models/lcm_dreamshaper_v7 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" ./image.png ./mask_image.png
 
+      - name: Run benchmark for text2image pipeline
+        run: |
+          source ./ov/setupvars.sh
+          ${{ env.build_dir }}/samples/cpp/image_generation/benchmark_image_gen -t text2image -m ./models/lcm_dreamshaper_v7 -p "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting"
+
+      - name: Run benchmark for image2image pipeline
+        run: |
+          source ./ov/setupvars.sh
+          ${{ env.build_dir }}/samples/cpp/image_generation/benchmark_image_gen -t image2image -m ./models/lcm_dreamshaper_v7 -p "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" -i ./image.png
+
+      - name: Run benchmark for inpainting
+        run: |
+          source ./ov/setupvars.sh
+          ${{ env.build_dir }}/samples/cpp/image_generation/benchmark_image_gen -t inpainting -m ./models/lcm_dreamshaper_v7 -p "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" -i ./image.png --mi ./mask_image.png
+
   lcm_dreamshaper_v7_cpp-windows:
     runs-on: windows-2022
     defaults:
@@ -126,7 +141,7 @@ jobs:
         run: |
           . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1"
           cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ${{ env.build_dir }}
-          cmake --build ${{ env.build_dir }} --config Release --target text2image image2image inpainting heterogeneous_stable_diffusion lora_text2image py_openvino_genai --parallel
+          cmake --build ${{ env.build_dir }} --config Release --target text2image image2image inpainting heterogeneous_stable_diffusion lora_text2image benchmark_image_gen py_openvino_genai --parallel
         env:
           CMAKE_TLS_VERIFY: 0
 
@@ -175,6 +190,33 @@ jobs:
         env:
           PYTHONPATH: ${{ env.build_dir }}
 
+      - name: Run benchmark_image_gen.py for text2image pipeline
+        run: |
+          . "./openvino_lcm_cpp/Scripts/Activate.ps1"
+          . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1"
+          $env:Path += "${{ env.build_dir }}\openvino_genai"
+          python .\samples\python\image_generation\benchmark_image_gen.py -t text2image -m .\models\lcm_dreamshaper_v7 -p"cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting"
+        env:
+          PYTHONPATH: ${{ env.build_dir }}
+
+      - name: Run benchmark_image_gen.py for image2image pipeline
+        run: |
+          . "./openvino_lcm_cpp/Scripts/Activate.ps1"
+          . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1"
+          $env:Path += "${{ env.build_dir }}\openvino_genai"
+          python .\samples\python\image_generation\benchmark_image_gen.py -t image2image -m .\models\lcm_dreamshaper_v7 -p"cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" -i .\image.png
+        env:
+          PYTHONPATH: ${{ env.build_dir }}
+
+      - name: Run benchmark_image_gen.py for inpainting pipeline
+        run: |
+          . "./openvino_lcm_cpp/Scripts/Activate.ps1"
+          . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1"
+          $env:Path += "${{ env.build_dir }}\openvino_genai"
+          python .\samples\python\image_generation\benchmark_image_gen.py -t inpainting -m .\models\lcm_dreamshaper_v7 -p"cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" -i .\image.png -mi .\mask_image.png
+        env:
+          PYTHONPATH: ${{ env.build_dir }}
+
   Overall_Status:
     name: ci/gha_overall_status_lcm
     needs: [lcm_dreamshaper_v7_cpp-linux, lcm_dreamshaper_v7_cpp-windows]

samples/cpp/image_generation/CMakeLists.txt

@@ -107,4 +107,30 @@ set_target_properties(inpainting PROPERTIES
 install(TARGETS inpainting
         RUNTIME DESTINATION samples_bin/
         COMPONENT samples_bin
-        EXCLUDE_FROM_ALL)
+        EXCLUDE_FROM_ALL)
+
+# benchmark_image_gen
+
+include(FetchContent)
+
+if(POLICY CMP0135)
+    cmake_policy(SET CMP0135 NEW)
+endif()
+
+FetchContent_Declare(cxxopts
+    URL https://github.com/jarro2783/cxxopts/archive/refs/tags/v3.1.1.tar.gz
+    URL_HASH SHA256=523175f792eb0ff04f9e653c90746c12655f10cb70f1d5e6d6d9491420298a08)
+FetchContent_MakeAvailable(cxxopts)
+
+add_executable(benchmark_image_gen benchmark_image_gen.cpp load_image.cpp imwrite.cpp)
+target_include_directories(benchmark_image_gen PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_BINARY_DIR}")
+target_link_libraries(benchmark_image_gen PRIVATE openvino::genai cxxopts::cxxopts indicators::indicators)
+set_target_properties(benchmark_image_gen PROPERTIES
+    COMPILE_PDB_NAME benchmark_image_gen
+    # Ensure out of box LC_RPATH on macOS with SIP
+    INSTALL_RPATH_USE_LINK_PATH ON)
+
+install(TARGETS benchmark_image_gen
+        RUNTIME DESTINATION samples_bin/
+        COMPONENT samples_bin
+        EXCLUDE_FROM_ALL)

samples/cpp/image_generation/README.md

@@ -8,6 +8,7 @@ There are several sample files:
  - [`heterogeneous_stable_diffusion.cpp`](./heterogeneous_stable_diffusion.cpp) shows how to assemble a heterogeneous txt2image pipeline from individual subcomponents (scheduler, text encoder, unet, vae decoder)
  - [`image2image.cpp`](./image2image.cpp) demonstrates basic usage of the image to image pipeline
  - [`inpainting.cpp`](./inpainting.cpp) demonstrates basic usage of the inpainting pipeline
+ - [`benchmark_image_gen.cpp`](./benchmark_image_gen.cpp) demonstrates how to benchmark the text to image / image to image / inpainting pipeline
 
 Users can change the sample code and play with the following generation parameters:
 
@@ -148,3 +149,63 @@ The resuling image is:
    ![](./inpainting.bmp)
 
 Note, that LoRA, heterogeneous execution and other features of `Text2ImagePipeline` are applicable for `InpaintingPipeline`.
+
+## benchmarking sample for image generation pipelines
+
+This `benchmark_image_gen.cpp` sample script demonstrates how to benchmark the text to image pipeline, image to image pipeline and inpainting pipeline. The script includes functionality for warm-up iterations, generating image, and calculating various performance metrics.
+
+The usage of this sample is:
+```bash
+./benchmark_image_gen [OPTIONS]
+```
+Options:
+- `-t, --pipeline_type` (default: `"text2image"`): Pipeline type(text2image, image2image, inpainting).
+- `-m, --model`: Path to the model and tokenizers base directory.
+- `-p, --prompt` (default: `"The Sky is blue because"`): The prompt to generate text.
+- `--nw, --num_warmup` (default: `1`): Number of warmup iterations.
+- `-n, --num_iter` (default: `3`): Number of iterations.
+- `-d, --device` (default: `"CPU"`): Device to run the model on.
+- `-w, --width` (default: `512`): The width of the output image.
+- `--ht, --height` (default: `512`): The height of the output image.
+- `--is, --num_inference_steps` (default: `20`): The number of inference steps.
+- `--ni, --num_images_per_prompt` (default: `1`): The number of images to generate per generate() call.
+- `-o, --output_dir` (default: `""`): Path to save output image.
+- `-i, --image`: Path to input image.
+- `-s, --strength`: Indicates extent to transform the reference `image`. Must be between 0 and 1.
+- `--mi, --mask_image`: Path to mask image.
+
+For example:
+
+`./benchmark_image_gen -t text2image -m dreamlike_anime_1_0_ov/FP16 -n 10 -d CPU`
+
+Performance output:
+
+```
+[warmup-0] generate time: 85008.00 ms, total infer time:84999.88 ms
+[warmup-0] text encoder infer time: 98.00 ms
+[warmup-0] unet iteration num:21, first iteration time:4317.94 ms, other iteration avg time:3800.91 ms
+[warmup-0] unet inference num:21, first inference time:4317.71 ms, other inference avg time:3800.61 ms
+[warmup-0] vae encoder infer time:0.00 ms, vae decoder infer time:4572.00 ms
+
+[iter-0] generate time: 84349.00 ms, total infer time:84340.97 ms
+[iter-0] text encoder infer time: 76.00 ms
+[iter-0] unet iteration num:21, first iteration time:3805.63 ms, other iteration avg time:3799.68 ms
+[iter-0] unet inference num:21, first inference time:3805.42 ms, other inference avg time:3799.38 ms
+[iter-0] vae encoder infer time:0.00 ms, vae decoder infer time:4472.00 ms
+
+[iter-1] generate time: 84391.00 ms, total infer time:84384.36 ms
+[iter-1] text encoder infer time: 78.00 ms
+[iter-1] unet iteration num:21, first iteration time:3801.15 ms, other iteration avg time:3802.17 ms
+[iter-1] unet inference num:21, first inference time:3800.93 ms, other inference avg time:3801.87 ms
+[iter-1] vae encoder infer time:0.00 ms, vae decoder infer time:4468.00 ms
+
+[iter-2] generate time: 84377.00 ms, total infer time:84366.51 ms
+[iter-2] text encoder infer time: 76.00 ms
+[iter-2] unet iteration num:21, first iteration time:3783.31 ms, other iteration avg time:3802.25 ms
+[iter-2] unet inference num:21, first inference time:3783.09 ms, other inference avg time:3801.82 ms
+[iter-2] vae encoder infer time:0.00 ms, vae decoder infer time:4471.00 ms
+
+Test finish, load time: 9356.00 ms
+Warmup number:1, first generate warmup time:85008.00 ms, infer warmup time:84999.88 ms
+Generate iteration number:3, for one iteration, generate avg time: 84372.34 ms, infer avg time:84363.95 ms, all text encoders infer avg time:76.67 ms, vae encoder infer avg time:0.00 ms, vae decoder infer avg time:4470.33 ms
+```