add performance statistics for image generation (#1405) #21
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: macOS (13, Python 3.10) | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| merge_group: | |
| push: | |
| branches: | |
| - master | |
| - 'releases/**' | |
| permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions | |
| concurrency: | |
| # github.ref is not unique in post-commit | |
| group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-mac | |
| cancel-in-progress: true | |
| env: | |
| MACOSX_DEPLOYMENT_TARGET: '11.0' | |
| PYTHON_VERSION: '3.10' | |
| TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }} | |
| CCACHE_MAXSIZE: 500Mi | |
| jobs: | |
| openvino_download: | |
| name: Download OpenVINO | |
| outputs: | |
| status: ${{ steps.openvino_download.outcome }} | |
| ov_artifact_name: ${{ steps.openvino_download.outputs.ov_artifact_name }} | |
| ov_wheel_source: ${{ steps.openvino_download.outputs.ov_wheel_source }} | |
| ov_version: ${{ steps.openvino_download.outputs.ov_version }} | |
| timeout-minutes: 10 | |
| defaults: | |
| run: | |
| shell: bash | |
| runs-on: aks-linux-2-cores-8gb | |
| container: | |
| image: 'openvinogithubactions.azurecr.io/openvino_provider:0.1.0' | |
| volumes: | |
| - /mount:/mount | |
| - ${{ github.workspace }}:${{ github.workspace }} | |
| steps: | |
| - uses: openvinotoolkit/openvino/.github/actions/openvino_provider@master | |
| id: openvino_download | |
| with: | |
| platform: macos_12_6 | |
| commit_packages_to_provide: wheels | |
| revision: latest_nightly | |
| genai_build_cmake: | |
| name: Build cpack - ${{ matrix.build-type }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| build-type: [Release] | |
| needs: [ openvino_download ] | |
| timeout-minutes: 20 | |
| defaults: | |
| run: | |
| shell: bash | |
| runs-on: macos-13 | |
| env: | |
| CMAKE_GENERATOR: Ninja | |
| CMAKE_CXX_COMPILER_LAUNCHER: ccache | |
| CMAKE_C_COMPILER_LAUNCHER: ccache | |
| CCACHE_DIR: ${{ github.workspace }}/ccache | |
| OV_INSTALL_DIR: ${{ github.workspace }}/ov | |
| INSTALL_DIR: ${{ github.workspace }}/install | |
| BUILD_DIR: ${{ github.workspace }}/build | |
| SRC_DIR: ${{ github.workspace }}/src | |
| steps: | |
| - name: Clone openvino.genai | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| path: ${{ env.SRC_DIR }} | |
| submodules: recursive | |
| - name: Download OpenVINO package | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| name: ${{ needs.openvino_download.outputs.ov_artifact_name }} | |
| path: ${{ env.OV_INSTALL_DIR }} | |
| merge-multiple: true | |
| - name: Install build dependencies | |
| run: brew install coreutils ninja ccache | |
| - name: Setup ccache | |
| id: ccache-restore | |
| uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 | |
| with: | |
| key: ${{ runner.os }}-${{ runner.arch }}-ccache-${{ env.TARGET_BRANCH }}-${{ github.sha }} | |
| restore-keys: | | |
| ${{ runner.os }}-${{ runner.arch }}-ccache-${{ env.TARGET_BRANCH }} | |
| path: ${{ env.CCACHE_DIR }} | |
| - name: Clean ccache stats | |
| run: ccache --zero-stats --show-config | |
| - name: CMake Build | |
| run: | | |
| source ${{ env.OV_INSTALL_DIR }}/setupvars.sh | |
| cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ${{ env.SRC_DIR}} -B ${{ env.BUILD_DIR }} | |
| cmake --build ${{ env.BUILD_DIR}} --config ${{ matrix.build-type }} --parallel $(nproc) --verbose | |
| cmake --install ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --prefix ${{ env.INSTALL_DIR }} | |
| - name: Show ccache stats | |
| run: ccache --show-stats | |
| - name: Save ccache | |
| if: always() && steps.ccache-restore.outputs.cache-hit != 'true' && github.event_name == 'push' | |
| uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 | |
| with: | |
| key: ${{ steps.ccache-restore.outputs.cache-primary-key }} | |
| path: ${{ env.CCACHE_DIR }} | |
| - name: Upload Archive Distribution Package | |
| if: always() | |
| uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
| with: | |
| name: genai_cpack_${{ matrix.build-type }} | |
| path: ${{ env.INSTALL_DIR }} | |
| if-no-files-found: 'error' | |
| genai_build_wheel: | |
| name: Build Wheel | |
| needs: [ openvino_download ] | |
| timeout-minutes: 20 | |
| defaults: | |
| run: | |
| shell: bash | |
| runs-on: macos-13 | |
| env: | |
| CCACHE_DIR: ${{ github.workspace }}/ccache | |
| CMAKE_CXX_COMPILER_LAUNCHER: ccache | |
| CMAKE_C_COMPILER_LAUNCHER: ccache | |
| OV_INSTALL_DIR: ${{ github.workspace }}/ov | |
| INSTALL_DIR: ${{ github.workspace }}/install | |
| WHEELS_DIR: ${{ github.workspace }}/install/wheels | |
| SRC_DIR: ${{ github.workspace }}/src | |
| steps: | |
| - name: Clone openvino.genai | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| path: ${{ env.SRC_DIR }} | |
| submodules: recursive | |
| - name: Download OpenVINO package | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| name: ${{ needs.openvino_download.outputs.ov_artifact_name }} | |
| path: ${{ env.OV_INSTALL_DIR }} | |
| merge-multiple: true | |
| - name: Setup Python ${{ env.PYTHON_VERSION }} | |
| uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| - name: Install build dependencies | |
| run: brew install coreutils ccache | |
| - name: Setup ccache | |
| id: ccache-restore | |
| uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 | |
| with: | |
| key: ${{ runner.os }}-${{ runner.arch }}-ccache-${{ env.TARGET_BRANCH }}-${{ github.sha }} | |
| restore-keys: | | |
| ${{ runner.os }}-${{ runner.arch }}-ccache-${{ env.TARGET_BRANCH }} | |
| path: ${{ env.CCACHE_DIR }} | |
| - name: Clean ccache stats | |
| run: ccache --zero-stats --show-config | |
| - name: Build Tokenizers Wheel | |
| run: | | |
| python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} \ | |
| ${{ needs.openvino_download.outputs.ov_wheel_source }} \ | |
| ${{ env.SRC_DIR }}/thirdparty/openvino_tokenizers | |
| working-directory: ${{ env.OV_INSTALL_DIR }} | |
| - name: Build GenAI Wheel | |
| run: | | |
| python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} \ | |
| ${{ needs.openvino_download.outputs.ov_wheel_source }} \ | |
| ${{ env.SRC_DIR }} | |
| working-directory: ${{ env.OV_INSTALL_DIR }} | |
| - name: Build WWB Wheel | |
| run: python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} ${{ env.SRC_DIR }}/tools/who_what_benchmark | |
| working-directory: ${{ env.OV_INSTALL_DIR }} | |
| - name: Save ccache | |
| if: always() && steps.ccache-restore.outputs.cache-hit != 'true' && github.event_name == 'push' | |
| uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 | |
| with: | |
| key: ${{ steps.ccache-restore.outputs.cache-primary-key }} | |
| path: ${{ env.CCACHE_DIR }} | |
| - name: Upload Wheels | |
| if: always() | |
| uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
| with: | |
| name: genai_wheels | |
| path: ${{ env.INSTALL_DIR }} | |
| if-no-files-found: 'error' | |
| genai_build_samples: | |
| name: Build Samples - ${{ matrix.build-type }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| build-type: [Release] | |
| needs: [ openvino_download, genai_build_cmake ] | |
| timeout-minutes: 10 | |
| defaults: | |
| run: | |
| shell: bash | |
| runs-on: macos-13 | |
| env: | |
| OV_INSTALL_DIR: ${{ github.workspace }}/ov | |
| INSTALL_DIR: ${{ github.workspace }}/install | |
| BUILD_DIR: ${{ github.workspace }}/build | |
| SRC_DIR: ${{ github.workspace }}/src | |
| steps: | |
| - name: Clone openvino.genai | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| path: ${{ env.SRC_DIR }} | |
| submodules: recursive | |
| - name: Download Build Artifacts | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| pattern: "{${{ needs.openvino_download.outputs.ov_artifact_name }},genai_cpack_${{ matrix.build-type }}}" | |
| path: ${{ env.OV_INSTALL_DIR }} | |
| merge-multiple: true | |
| - name: Build Samples (Release) | |
| if: ${{ 'Release' == matrix.build-type }} | |
| run: | | |
| chmod +x ${{ env.OV_INSTALL_DIR }}/samples/cpp/build_samples.sh | |
| ${{ env.OV_INSTALL_DIR }}/samples/cpp/build_samples.sh -i ${{ env.INSTALL_DIR }} | |
| - name: Build Samples (${{ matrix.build-type }}) | |
| if: ${{ 'Release' != matrix.build-type }} | |
| run: | | |
| source ${{ env.OV_INSTALL_DIR }}/setupvars.sh | |
| cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ${{ env.OV_INSTALL_DIR }}/samples/cpp/ -B ${{ env.BUILD_DIR }} | |
| cmake --build ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --parallel $(nproc) | |
| cmake --install ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --component samples_bin --prefix ${{ env.INSTALL_DIR }} | |
| - name: Upload Samples Build Package | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
| with: | |
| name: genai_samples_${{ matrix.build-type }} | |
| path: ${{ env.INSTALL_DIR }} | |
| if-no-files-found: 'error' | |
| genai_tests_wheel: | |
| name: Python (${{ matrix.test.name}}) Tests (wheel) | |
| needs: [ openvino_download, genai_build_wheel ] | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test: | |
| - name: 'Whisper' | |
| cmd: 'tests/python_tests/test_whisper_pipeline.py' | |
| - name: 'Cacheopt E2E' | |
| cmd: 'tests/python_tests/test_kv_cache_eviction.py' | |
| - name: 'LLM & VLM' | |
| cmd: 'tests/python_tests --ignore tests/python_tests/test_whisper_pipeline.py --ignore tests/python_tests/test_kv_cache_eviction.py --ignore tests/python_tests/test_whisper_pipeline_static.py' | |
| defaults: | |
| run: | |
| shell: bash | |
| runs-on: macos-13 | |
| env: | |
| INSTALL_DIR: ${{ github.workspace }}/install | |
| SRC_DIR: ${{ github.workspace }}/src | |
| BUILD_DIR: ${{ github.workspace }}/build | |
| TRANSFORMERS_CACHE: ${{ github.workspace }}/models # Hugging Face transformers cache | |
| HF_HOME: ${{ github.workspace }}/datasets # Hugging Face datasets cache | |
| steps: | |
| - name: Clone openvino.genai | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| path: ${{ env.SRC_DIR }} | |
| submodules: recursive | |
| - name: Download Build Artifacts | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| pattern: "{${{ needs.openvino_download.outputs.ov_artifact_name }},genai_wheels}" | |
| path: ${{ env.INSTALL_DIR }} | |
| merge-multiple: true | |
| - name: Setup Python ${{ env.PYTHON_VERSION }} | |
| uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| - name: Install OpenVINO wheel | |
| run: | | |
| python3 -m pip install openvino==${{ needs.openvino_download.outputs.ov_version }} ${{ needs.openvino_download.outputs.ov_wheel_source }} | |
| working-directory: ${{ env.INSTALL_DIR }} | |
| - name: Install GenAI Wheels | |
| uses: ./src/.github/actions/install_wheel | |
| with: | |
| packages: "openvino_tokenizers[transformers];openvino_genai;whowhatbench" | |
| requirements_files: "${{ env.SRC_DIR }}/tests/python_tests/requirements.txt" | |
| local_wheel_dir: ${{ env.INSTALL_DIR }}/wheels | |
| - name: Tests | |
| run: python -m pytest -v ./${{ matrix.test.cmd }} | |
| working-directory: ${{ env.SRC_DIR }} | |
| genai_samples_tests: | |
| name: Samples Tests - ${{ matrix.build-type }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| build-type: [Release] | |
| needs: [ openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples ] | |
| timeout-minutes: 45 | |
| defaults: | |
| run: | |
| shell: bash | |
| runs-on: macos-13 | |
| env: | |
| INSTALL_DIR: ${{ github.workspace }}/ov | |
| SRC_DIR: ${{ github.workspace }}/src | |
| BUILD_DIR: ${{ github.workspace }}/build | |
| MODELS_DIR: ${{ github.workspace }}/models | |
| steps: | |
| - name: Clone openvino.genai | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| path: ${{ env.SRC_DIR }} | |
| submodules: recursive | |
| - name: Download Build Artifacts | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| pattern: "{${{ needs.openvino_download.outputs.ov_artifact_name }},genai_cpack_${{ matrix.build-type }},genai_samples_${{ matrix.build-type }},genai_wheels}" | |
| path: ${{ env.INSTALL_DIR }} | |
| merge-multiple: true | |
| - name: Setup Python ${{ env.PYTHON_VERSION }} | |
| uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| - name: Install OpenVINO wheel | |
| run: | | |
| python3 -m pip install openvino==${{ needs.openvino_download.outputs.ov_version }} ${{ needs.openvino_download.outputs.ov_wheel_source }} | |
| working-directory: ${{ env.INSTALL_DIR }} | |
| - name: Install GenAI wheels | |
| uses: ./src/.github/actions/install_wheel | |
| with: | |
| packages: "openvino_tokenizers[transformers];openvino_genai" | |
| requirements_files: "${{ env.SRC_DIR }}/samples/requirements.txt" | |
| local_wheel_dir: ${{ env.INSTALL_DIR }}/wheels | |
| - name: Download & convert Models and data | |
| run: | | |
| mkdir -p ${{ env.MODELS_DIR }} | |
| optimum-cli export openvino --trust-remote-code --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 ${{ env.MODELS_DIR }}/TinyLlama-1.1B-Chat-v1.0 | |
| optimum-cli export openvino --trust-remote-code --model openai/whisper-tiny ${{ env.MODELS_DIR }}/whisper-tiny | |
| wget https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/librispeech_s5/how_are_you_doing_today.wav -O ${{ env.MODELS_DIR }}/how_are_you_doing_today.wav | |
| - name: Test multinomial_causal_lm.py | |
| if: ${{ 'Release' == matrix.build-type }} # Python bindings can be built in Release only | |
| timeout-minutes: 1 | |
| run: python ${{ env.INSTALL_DIR }}/samples/python/text_generation/multinomial_causal_lm.py ./TinyLlama-1.1B-Chat-v1.0/ 0 | |
| working-directory: ${{ env.MODELS_DIR }} | |
| - name: Test whisper_speech_recognition.py | |
| if: ${{ 'Release' == matrix.build-type }} # Python bindings can be built in Release only | |
| timeout-minutes: 1 | |
| run: python ${{ env.INSTALL_DIR }}/samples/python/whisper_speech_recognition/whisper_speech_recognition.py ./whisper-tiny/ how_are_you_doing_today.wav | |
| working-directory: ${{ env.MODELS_DIR }} | |
| - name: C++ Tests Prerequisites | |
| run: python -m pip uninstall openvino openvino-tokenizers openvino-genai -y | |
| - name: Test greedy_causal_lm | |
| run: | | |
| source ${{ env.INSTALL_DIR }}/setupvars.sh | |
| chmod +x ${{ env.INSTALL_DIR }}/samples_bin/greedy_causal_lm | |
| ${{ env.INSTALL_DIR }}/samples_bin/greedy_causal_lm ./TinyLlama-1.1B-Chat-v1.0/ "" | |
| working-directory: ${{ env.MODELS_DIR }} | |
| - name: Test whisper_speech_recognition | |
| run: | | |
| source ${{ env.INSTALL_DIR }}/setupvars.sh | |
| chmod +x ${{ env.INSTALL_DIR }}/samples_bin/whisper_speech_recognition | |
| ${{ env.INSTALL_DIR }}/samples_bin/whisper_speech_recognition ./whisper-tiny/ how_are_you_doing_today.wav | |
| working-directory: ${{ env.MODELS_DIR }} | |
| Overall_Status: | |
| name: ci/gha_overall_status_macos | |
| needs: [openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples, genai_tests_wheel, genai_samples_tests] | |
| if: ${{ always() }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check status of all jobs | |
| if: >- | |
| ${{ | |
| contains(needs.*.result, 'failure') || | |
| contains(needs.*.result, 'cancelled') | |
| }} | |
| run: exit 1 |