Switch NPU LLM execution to ov::genai::StatefulLLMPipeline (#1677) #22
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Linux (Ubuntu 22.04, Python 3.11) | |
on: | |
workflow_dispatch: | |
pull_request: | |
merge_group: | |
push: | |
branches: | |
- master | |
- 'releases/**' | |
permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions | |
concurrency: | |
# github.ref is not unique in post-commit | |
group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-linux | |
cancel-in-progress: true | |
env: | |
PYTHON_VERSION: '3.11' | |
OV_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }} | |
CMAKE_CXX_COMPILER_LAUNCHER: sccache | |
CMAKE_C_COMPILER_LAUNCHER: sccache | |
SCCACHE_IGNORE_SERVER_IO_ERROR: 1 | |
SCCACHE_SERVER_PORT: 35555 | |
SCCACHE_CACHE_SIZE: 30G | |
SCCACHE_AZURE_KEY_PREFIX: genai/ubuntu/22_04/x64 | |
GENAI_ARCHIVE_NAME: genai.tar.gz | |
GENAI_SAMPLES_NAME: genai_samples.tar.gz | |
jobs: | |
openvino_download: | |
name: Download OpenVINO | |
outputs: | |
status: ${{ steps.openvino_download.outcome }} | |
ov_artifact_name: ${{ steps.openvino_download.outputs.ov_artifact_name }} | |
ov_wheel_source: ${{ steps.openvino_download.outputs.ov_wheel_source }} | |
docker_tag: ${{ steps.get_docker_tag.outputs.docker_tag }} | |
timeout-minutes: 10 | |
defaults: | |
run: | |
shell: bash | |
runs-on: aks-linux-2-cores-8gb | |
container: | |
image: 'openvinogithubactions.azurecr.io/openvino_provider:0.1.0' | |
volumes: | |
- /mount:/mount | |
- ${{ github.workspace }}:${{ github.workspace }} | |
steps: | |
- uses: openvinotoolkit/openvino/.github/actions/openvino_provider@master | |
id: openvino_download | |
with: | |
platform: ubuntu22 | |
commit_packages_to_provide: wheels,openvino_js_package.tar.gz | |
revision: a8aba4e104f027c2ba8a21fd6c4c861110c57ed9 | |
- name: Clone docker tag from OpenVINO repo | |
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
with: | |
repository: 'openvinotoolkit/openvino' | |
path: 'openvino' | |
ref: ${{ env.OV_BRANCH }} | |
sparse-checkout: | | |
.github/dockerfiles/docker_tag | |
- name: Save docker tag to output | |
id: get_docker_tag | |
run: | | |
docker_tag=$(cat openvino/.github/dockerfiles/docker_tag) | |
echo "docker_tag=$docker_tag" >> $GITHUB_OUTPUT | |
genai_build_cmake: | |
name: Build Archive - ${{ matrix.build-type }} | |
strategy: | |
fail-fast: false | |
matrix: | |
build-type: [Release] | |
needs: [ openvino_download ] | |
timeout-minutes: 20 | |
defaults: | |
run: | |
shell: bash | |
runs-on: aks-linux-4-cores-16gb | |
container: | |
image: openvinogithubactions.azurecr.io/ov_build/ubuntu_22_04_x64:${{ needs.openvino_download.outputs.docker_tag }} | |
volumes: | |
- /mount:/mount | |
- ${{ github.workspace }}:${{ github.workspace }} | |
options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING | |
env: | |
CMAKE_GENERATOR: Unix Makefiles | |
OV_INSTALL_DIR: ${{ github.workspace }}/ov | |
INSTALL_DIR: ${{ github.workspace }}/install | |
INSTALL_SAMPLES_DIR: ${{ github.workspace }}/samples | |
BUILD_DIR: ${{ github.workspace }}/build | |
SRC_DIR: ${{ github.workspace }}/src | |
steps: | |
- name: Clone openvino.genai | |
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
with: | |
path: ${{ env.SRC_DIR }} | |
submodules: recursive | |
- name: Download OpenVINO package | |
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
with: | |
name: ${{ needs.openvino_download.outputs.ov_artifact_name }} | |
path: ${{ env.OV_INSTALL_DIR }} | |
merge-multiple: true | |
- name: CMake Build | |
run: | | |
source ${{ env.OV_INSTALL_DIR }}/setupvars.sh | |
cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ${{ env.SRC_DIR}} -B ${{ env.BUILD_DIR }} | |
cmake --build ${{ env.BUILD_DIR}} --config ${{ matrix.build-type }} --parallel $(nproc) --verbose | |
cmake --install ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --prefix ${{ env.INSTALL_DIR }} | |
- name: Pack Artifacts | |
run: tar -cvf - * | pigz > ${{ env.BUILD_DIR }}/${{ env.GENAI_ARCHIVE_NAME }} | |
working-directory: ${{ env.INSTALL_DIR }} | |
- name: Upload Archive Distribution Package | |
if: ${{ always() }} | |
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
with: | |
name: genai_archive_${{ matrix.build-type }} | |
path: ${{ env.BUILD_DIR }}/${{ env.GENAI_ARCHIVE_NAME }} | |
if-no-files-found: 'error' | |
genai_build_wheel: | |
name: Build Wheel | |
needs: [ openvino_download ] | |
timeout-minutes: 20 | |
defaults: | |
run: | |
shell: bash | |
runs-on: aks-linux-4-cores-16gb | |
container: | |
image: openvinogithubactions.azurecr.io/ov_build/ubuntu_22_04_x64:${{ needs.openvino_download.outputs.docker_tag }} | |
volumes: | |
- /mount:/mount | |
- ${{ github.workspace }}:${{ github.workspace }} | |
options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING | |
env: | |
CMAKE_GENERATOR: Unix Makefiles | |
OV_INSTALL_DIR: ${{ github.workspace }}/ov | |
INSTALL_DIR: ${{ github.workspace }}/install | |
WHEELS_DIR: ${{ github.workspace }}/install/wheels | |
SRC_DIR: ${{ github.workspace }}/src | |
steps: | |
- name: Clone openvino.genai | |
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
with: | |
path: ${{ env.SRC_DIR }} | |
submodules: recursive | |
- name: Download OpenVINO package | |
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
with: | |
name: ${{ needs.openvino_download.outputs.ov_artifact_name }} | |
path: ${{ env.OV_INSTALL_DIR }} | |
merge-multiple: true | |
- name: Build Tokenizers Wheel | |
run: | | |
python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} \ | |
--config-settings=override=cross.arch="manylinux_2_31_x86_64" \ | |
${{ needs.openvino_download.outputs.ov_wheel_source }} \ | |
${{ env.SRC_DIR }}/thirdparty/openvino_tokenizers | |
working-directory: ${{ env.OV_INSTALL_DIR }} | |
- name: Build GenAI Wheel | |
run: | | |
python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} \ | |
--config-settings=override=cross.arch="manylinux_2_31_x86_64" \ | |
${{ needs.openvino_download.outputs.ov_wheel_source }} \ | |
${{ env.SRC_DIR }} | |
working-directory: ${{ env.OV_INSTALL_DIR }} | |
- name: Build WWB Wheel | |
run: python -m pip wheel -v --no-deps --wheel-dir ${{ env.WHEELS_DIR }} ${{ env.SRC_DIR }}/tools/who_what_benchmark | |
working-directory: ${{ env.OV_INSTALL_DIR }} | |
- name: Upload Wheels | |
if: ${{ always() }} | |
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
with: | |
name: genai_wheels | |
path: ${{ env.INSTALL_DIR }} | |
if-no-files-found: 'error' | |
genai_build_samples: | |
name: Build Samples - ${{ matrix.build-type }} | |
strategy: | |
fail-fast: false | |
matrix: | |
build-type: [Release] | |
needs: [ openvino_download, genai_build_cmake ] | |
timeout-minutes: 10 | |
defaults: | |
run: | |
shell: bash | |
runs-on: aks-linux-4-cores-16gb | |
container: | |
image: openvinogithubactions.azurecr.io/ov_build/ubuntu_22_04_x64:${{ needs.openvino_download.outputs.docker_tag }} | |
volumes: | |
- /mount:/mount | |
- ${{ github.workspace }}:${{ github.workspace }} | |
options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING | |
env: | |
CMAKE_GENERATOR: Unix Makefiles | |
OV_INSTALL_DIR: ${{ github.workspace }}/ov | |
INSTALL_DIR: ${{ github.workspace }}/install | |
BUILD_DIR: ${{ github.workspace }}/build | |
SRC_DIR: ${{ github.workspace }}/src | |
steps: | |
- name: Clone openvino.genai | |
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
with: | |
path: ${{ env.SRC_DIR }} | |
submodules: recursive | |
- name: Download Build Artifacts | |
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
with: | |
pattern: "{${{ needs.openvino_download.outputs.ov_artifact_name }},genai_archive_${{ matrix.build-type }}}" | |
path: ${{ env.OV_INSTALL_DIR }} | |
merge-multiple: true | |
- name: Extract Artifacts | |
run: pigz -dc ${{ env.GENAI_ARCHIVE_NAME }} | tar -xf - -C ${{ env.OV_INSTALL_DIR }} | |
working-directory: ${{ env.OV_INSTALL_DIR }} | |
- name: Build Samples (Release) | |
if: ${{ 'Release' == matrix.build-type }} | |
run: | | |
chmod +x ${{ env.OV_INSTALL_DIR }}/samples/cpp/build_samples.sh | |
${{ env.OV_INSTALL_DIR }}/samples/cpp/build_samples.sh -i ${{ env.INSTALL_DIR }} | |
- name: Build Samples (${{ matrix.build-type }}) | |
if: ${{ 'Release' != matrix.build-type }} | |
run: | | |
source ${{ env.OV_INSTALL_DIR }}/setupvars.sh | |
cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ${{ env.OV_INSTALL_DIR }}/samples/cpp/ -B ${{ env.BUILD_DIR }} | |
cmake --build ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --parallel $(nproc) | |
cmake --install ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --component samples_bin --prefix ${{ env.INSTALL_DIR }} | |
- name: Pack Artifacts | |
run: tar -cvf - * | pigz > ${{ env.INSTALL_DIR }}/${{ env.GENAI_SAMPLES_NAME }} | |
working-directory: ${{ env.INSTALL_DIR }} | |
- name: Upload Samples Build Package | |
if: ${{ always() }} | |
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
with: | |
name: genai_samples_${{ matrix.build-type }} | |
path: ${{ env.INSTALL_DIR }}/*.tar.gz | |
if-no-files-found: 'error' | |
genai_tests_wheel: | |
name: Python (${{ matrix.test.name}}) Tests (wheel) | |
needs: [ openvino_download, genai_build_wheel ] | |
timeout-minutes: 60 | |
strategy: | |
fail-fast: false | |
matrix: | |
test: | |
- name: 'Whisper' | |
cmd: 'tests/python_tests/test_whisper_pipeline.py tests/python_tests/test_whisper_pipeline_static.py' | |
- name: 'Cacheopt E2E' | |
cmd: 'tests/python_tests/test_kv_cache_eviction.py' | |
- name: 'LLM & VLM' | |
cmd: 'tests/python_tests --ignore tests/python_tests/test_whisper_pipeline.py --ignore tests/python_tests/test_kv_cache_eviction.py --ignore tests/python_tests/test_whisper_pipeline_static.py' | |
defaults: | |
run: | |
shell: bash | |
runs-on: aks-linux-4-cores-16gb | |
container: | |
image: openvinogithubactions.azurecr.io/ov_test/ubuntu_22_04_x64:${{ needs.openvino_download.outputs.docker_tag }} | |
volumes: | |
- /mount:/mount | |
- ${{ github.workspace }}:${{ github.workspace }} | |
env: | |
INSTALL_DIR: ${{ github.workspace }}/install | |
SRC_DIR: ${{ github.workspace }}/src | |
BUILD_DIR: ${{ github.workspace }}/build | |
TRANSFORMERS_CACHE: ${{ github.workspace }}/models # Hugging Face transformers cache | |
HF_HOME: ${{ github.workspace }}/datasets # Hugging Face datasets cache | |
steps: | |
- name: Clone openvino.genai | |
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
with: | |
path: ${{ env.SRC_DIR }} | |
submodules: recursive | |
- name: Download Build Artifacts | |
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
with: | |
pattern: "{${{ needs.openvino_download.outputs.ov_artifact_name }},genai_wheels}" | |
path: ${{ env.INSTALL_DIR }} | |
merge-multiple: true | |
- name: Install GenAI Wheels | |
uses: ./src/.github/actions/install_wheel | |
with: | |
packages: "openvino;openvino_tokenizers[transformers];openvino_genai;whowhatbench" | |
requirements_files: "${{ env.SRC_DIR }}/tests/python_tests/requirements.txt" | |
local_wheel_dir: ${{ env.INSTALL_DIR }}/wheels | |
- name: Tests | |
run: python -m pytest -v ./${{ matrix.test.cmd }} | |
working-directory: ${{ env.SRC_DIR }} | |
genai_samples_tests: | |
name: Samples ${{ matrix.test.name }} (${{ matrix.build-type }}) | |
strategy: | |
fail-fast: false | |
matrix: | |
build-type: [Release] | |
test: | |
- name: 'LLM' | |
marker: 'llm' | |
cmd: 'tests/python_tests/samples' | |
- name: 'Whisper' | |
marker: 'whisper' | |
cmd: 'tests/python_tests/samples' | |
needs: [ openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples ] | |
timeout-minutes: 45 | |
defaults: | |
run: | |
shell: bash | |
runs-on: aks-linux-4-cores-16gb | |
container: | |
image: openvinogithubactions.azurecr.io/ov_test/ubuntu_22_04_x64:${{ needs.openvino_download.outputs.docker_tag }} | |
volumes: | |
- /mount:/mount | |
- ${{ github.workspace }}:${{ github.workspace }} | |
env: | |
INSTALL_DIR: ${{ github.workspace }}/ov | |
SRC_DIR: ${{ github.workspace }}/src | |
BUILD_DIR: ${{ github.workspace }}/build | |
steps: | |
- name: Clone openvino.genai | |
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
with: | |
path: ${{ env.SRC_DIR }} | |
submodules: recursive | |
- name: Download Build Artifacts | |
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
with: | |
pattern: "{${{ needs.openvino_download.outputs.ov_artifact_name }},genai_archive_${{ matrix.build-type }},genai_samples_${{ matrix.build-type }},genai_wheels}" | |
path: ${{ env.INSTALL_DIR }} | |
merge-multiple: true | |
- name: Extract Artifacts | |
run: | | |
pigz -dc ${{ env.GENAI_ARCHIVE_NAME }} | tar -xf - -C ${{ env.INSTALL_DIR }} | |
pigz -dc ${{ env.GENAI_SAMPLES_NAME }} | tar -xf - -C ${{ env.INSTALL_DIR }} | |
working-directory: ${{ env.INSTALL_DIR }} | |
- name: Install Wheels | |
uses: ./src/.github/actions/install_wheel | |
with: | |
packages: "openvino;openvino_tokenizers[transformers];openvino_genai[testing]" | |
requirements_files: "${{ env.SRC_DIR }}/samples/requirements.txt" | |
local_wheel_dir: ${{ env.INSTALL_DIR }}/wheels | |
- name: Test Samples (Python and C++) | |
run: python -m pytest -v ${{ env.SRC_DIR }}/${{ matrix.test.cmd }} -m "${{ matrix.test.marker }}" | |
env: | |
LD_LIBRARY_PATH: "${{ env.INSTALL_DIR }}/runtime/lib/intel64:${{ env.INSTALL_DIR }}/runtime/3rdparty/tbb/lib:$LD_LIBRARY_PATH" # Required for C++ samples | |
SAMPLES_PY_DIR: "${{ env.INSTALL_DIR }}/samples/python" | |
SAMPLES_CPP_DIR: "${{ env.INSTALL_DIR }}/samples_bin" | |
genai_build_nodejs_bindings: | |
name: Build Node.js bindings | |
strategy: | |
fail-fast: false | |
matrix: | |
build-type: [Release] | |
needs: [ openvino_download ] | |
timeout-minutes: 20 | |
defaults: | |
run: | |
shell: bash | |
runs-on: aks-linux-4-cores-16gb | |
container: | |
image: openvinogithubactions.azurecr.io/ov_build/ubuntu_22_04_x64:${{ needs.openvino_download.outputs.docker_tag }} | |
volumes: | |
- /mount:/mount | |
options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING -v ${{ github.workspace }}:${{ github.workspace }} | |
env: | |
CMAKE_GENERATOR: Unix Makefiles | |
OV_INSTALL_DIR: ${{ github.workspace }}/ov | |
SRC_DIR: ${{ github.workspace }}/openvino.genai | |
INSTALL_DIR: ${{ github.workspace }}/openvino.genai/src/js/bin | |
BUILD_DIR: ${{ github.workspace }}/build | |
steps: | |
- name: Clone openvino.genai | |
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
with: | |
path: ${{ env.SRC_DIR }} | |
submodules: recursive | |
- name: Download OpenVINO package | |
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
with: | |
name: ${{ needs.openvino_download.outputs.ov_artifact_name }} | |
path: ${{ env.OV_INSTALL_DIR }} | |
merge-multiple: true | |
- name: Build GenAI Node.js bindings | |
run: | | |
source ${{ env.OV_INSTALL_DIR }}/setupvars.sh | |
cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \ | |
-DENABLE_JS=ON -DCPACK_GENERATOR=NPM \ | |
-DENABLE_PYTHON=OFF -DENABLE_WHEEL=OFF \ | |
-S ${{ env.SRC_DIR }} -B ${{ env.BUILD_DIR }} | |
cmake --build ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --parallel $(nproc) --verbose | |
cmake --install ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --prefix ${{ env.INSTALL_DIR }} | |
- name: Pack Node.js bindings libs | |
run: tar -cvf - * | pigz > ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz | |
working-directory: ${{ env.INSTALL_DIR }} | |
- name: Upload Archive Package with Node.js bindings | |
if: ${{ always() }} | |
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
with: | |
name: genai_nodejs_bindings | |
path: ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz | |
if-no-files-found: 'error' | |
# JS pacakges uses the OpenVINO and OpenVINO GenAI libraries from the bin directory. | |
# Here we emulate the installation of the openvino-node package from NPM. The latest | |
# release of the openvino-node package is installed, and we need to update the binaries | |
# in the node_modules/openvino-node/bin directory to work correctly with GenAI | |
- name: Install npm package tests dependencies | |
working-directory: ${{ env.SRC_DIR }}/src/js | |
run: | | |
npm install --verbose | |
rm -rf node_modules/openvino-node/bin | |
cp -R ${{ env.OV_INSTALL_DIR }}/openvino_js_package node_modules/openvino-node/bin | |
- name: Check lint | |
working-directory: ${{ env.SRC_DIR }}/src/js | |
run: npm run lint | |
- name: Run npm package tests | |
working-directory: ${{ env.SRC_DIR }}/src/js | |
run: npm test | |
- name: Install openvino-genai-node samples dependencies | |
working-directory: ${{ env.SRC_DIR }}/samples/js/text_generation | |
run: | | |
npm install --verbose | |
rm -rf node_modules/openvino-node/bin | |
cp -R ${{ env.OV_INSTALL_DIR }}/openvino_js_package node_modules/openvino-node/bin | |
- name: Run samples tests | |
run: npm test | |
env: | |
MODEL_PATH: ${{ env.SRC_DIR }}/src/js/tests/models/Llama-3.1-8B-Instruct-FastDraft-150M-int8-ov | |
working-directory: ${{ env.SRC_DIR }}/samples/js/text_generation | |
Overall_Status: | |
name: ci/gha_overall_status_linux | |
needs: [openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples, genai_tests_wheel, genai_samples_tests, genai_build_nodejs_bindings] | |
if: ${{ always() }} | |
runs-on: ubuntu-latest | |
steps: | |
- name: Check status of all jobs | |
if: >- | |
${{ | |
contains(needs.*.result, 'failure') || | |
contains(needs.*.result, 'cancelled') | |
}} | |
run: exit 1 |