Updated to recent OV / Tokenizers state (openvinotoolkit#805)

yatarkan · Aug 28, 2024 · d60455d · d60455d
1 parent 2cc8771
commit d60455d
Show file tree

Hide file tree

Showing 7 changed files with 23 additions and 21 deletions.
diff --git a/.github/workflows/causal_lm_cpp.yml b/.github/workflows/causal_lm_cpp.yml
@@ -13,9 +13,9 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  l_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2024.4.0-16449-1f3edda195c/l_openvino_toolkit_ubuntu20_2024.4.0.dev20240823_x86_64.tgz
-  m_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2024.4.0-16449-1f3edda195c/m_openvino_toolkit_macos_12_6_2024.4.0.dev20240823_x86_64.tgz
-  w_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2024.4.0-16449-1f3edda195c/w_openvino_toolkit_windows_2024.4.0.dev20240823_x86_64.zip
+  l_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2024.4.0-16527-382ac845923/l_openvino_toolkit_ubuntu20_2024.4.0.dev20240828_x86_64.tgz
+  m_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2024.4.0-16527-382ac845923/m_openvino_toolkit_macos_12_6_2024.4.0.dev20240828_x86_64.tgz
+  w_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2024.4.0-16527-382ac845923/w_openvino_toolkit_windows_2024.4.0.dev20240828_x86_64.zip
 jobs:
   cpp-multinomial-greedy_causal_lm-ubuntu:
     runs-on: ubuntu-20.04-8-cores
@@ -41,17 +41,17 @@ jobs:
           cmake --build ./build/ --config Release -j
       - run: >
           . ./ov/setupvars.sh
-          && PYTHONPATH=./build/:$PYTHONPATH timeout 50s
+          && PYTHONPATH=./build/:$PYTHONPATH timeout 25s
           ./build/samples/cpp/multinomial_causal_lm/multinomial_causal_lm ./open_llama_3b_v2/ a
       - run: >
           . ./ov/setupvars.sh
-          && PYTHONPATH=./build/:$PYTHONPATH timeout 50s
+          && PYTHONPATH=./build/:$PYTHONPATH timeout 25s
           ./samples/python/multinomial_causal_lm/multinomial_causal_lm.py ./open_llama_3b_v2/ b
       - run: >
           . ./ov/setupvars.sh
           && export PYTHONPATH=./build/:$PYTHONPATH
-          && timeout 50s ./build/samples/cpp/greedy_causal_lm/greedy_causal_lm ./open_llama_3b_v2/ "return 0"
-          | diff <(timeout 50s samples/python/greedy_causal_lm/greedy_causal_lm.py ./open_llama_3b_v2/ "return 0") -
+          && timeout 25s ./build/samples/cpp/greedy_causal_lm/greedy_causal_lm ./open_llama_3b_v2/ "return 0"
+          | diff <(timeout 25s samples/python/greedy_causal_lm/greedy_causal_lm.py ./open_llama_3b_v2/ "return 0") -
 
   cpp-beam_search_causal_lm-ubuntu:
     strategy:

diff --git a/.github/workflows/lcm_dreamshaper_cpp.yml b/.github/workflows/lcm_dreamshaper_cpp.yml
@@ -16,8 +16,8 @@ permissions: read-all  # Required by https://github.com/ossf/scorecard/blob/e23b
 env:
   WORKING_DIRECTORY: "./image_generation/lcm_dreamshaper_v7/cpp/"
   PYTHON_VERSION: '3.8'
-  LINUX_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2024.4.0-16449-1f3edda195c/l_openvino_toolkit_ubuntu20_2024.4.0.dev20240823_x86_64.tgz
-  WINDOWS_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2024.4.0-16449-1f3edda195c/w_openvino_toolkit_windows_2024.4.0.dev20240823_x86_64.zip
+  LINUX_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2024.4.0-16527-382ac845923/l_openvino_toolkit_ubuntu20_2024.4.0.dev20240828_x86_64.tgz
+  WINDOWS_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2024.4.0-16527-382ac845923/w_openvino_toolkit_windows_2024.4.0.dev20240828_x86_64.zip
   OV_INSTALL_DIR: ${{ github.workspace }}/ov
 
 concurrency:

diff --git a/.github/workflows/stable_diffusion_1_5_cpp.yml b/.github/workflows/stable_diffusion_1_5_cpp.yml
@@ -16,8 +16,8 @@ permissions: read-all  # Required by https://github.com/ossf/scorecard/blob/e23b
 env:
   WORKING_DIRECTORY: "./image_generation/stable_diffusion_1_5/cpp/"
   PYTHON_VERSION: '3.8'
-  LINUX_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2024.4.0-16449-1f3edda195c/l_openvino_toolkit_ubuntu20_2024.4.0.dev20240823_x86_64.tgz
-  WINDOWS_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2024.4.0-16449-1f3edda195c/w_openvino_toolkit_windows_2024.4.0.dev20240823_x86_64.zip
+  LINUX_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2024.4.0-16527-382ac845923/l_openvino_toolkit_ubuntu20_2024.4.0.dev20240828_x86_64.tgz
+  WINDOWS_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2024.4.0-16527-382ac845923/w_openvino_toolkit_windows_2024.4.0.dev20240828_x86_64.zip
   OV_INSTALL_DIR: ${{ github.workspace }}/ov
 
 concurrency:

diff --git a/samples/cpp/prompt_lookup_decoding_lm/CMakeLists.txt b/samples/cpp/prompt_lookup_decoding_lm/CMakeLists.txt
@@ -9,7 +9,7 @@ find_package(OpenVINOGenAI REQUIRED PATHS
 )
 
 add_executable(prompt_lookup_decoding_lm prompt_lookup_decoding_lm.cpp)
-target_link_libraries(prompt_lookup_decoding_lm PRIVATE openvino::runtime)
+target_link_libraries(prompt_lookup_decoding_lm PRIVATE openvino::runtime openvino::threading)
 set_target_properties(prompt_lookup_decoding_lm PROPERTIES
     COMPILE_PDB_NAME prompt_lookup_decoding_lm
     # Ensure out of box LC_RPATH on macOS with SIP

diff --git a/samples/cpp/speculative_decoding_lm/CMakeLists.txt b/samples/cpp/speculative_decoding_lm/CMakeLists.txt
@@ -9,7 +9,7 @@ find_package(OpenVINOGenAI REQUIRED PATHS
 )
 
 add_executable(speculative_decoding_lm speculative_decoding_lm.cpp)
-target_link_libraries(speculative_decoding_lm PRIVATE openvino::runtime)
+target_link_libraries(speculative_decoding_lm PRIVATE openvino::runtime openvino::threading)
 set_target_properties(speculative_decoding_lm PROPERTIES
     COMPILE_PDB_NAME speculative_decoding_lm
     # Ensure out of box LC_RPATH on macOS with SIP

diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt
@@ -2,14 +2,16 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-set(INSTALL_GTEST OFF CACHE BOOL "")
+if(NOT TARGET gtest)
+  set(INSTALL_GTEST OFF CACHE BOOL "")
 
-FetchContent_Declare(
-  googletest
-  URL       https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip
-  URL_HASH SHA256=edd885a1ab32b6999515a880f669efadb80b3f880215f315985fa3f6eca7c4d3
-)
-FetchContent_MakeAvailable(googletest)
+  FetchContent_Declare(
+    googletest
+    URL       https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip
+    URL_HASH SHA256=edd885a1ab32b6999515a880f669efadb80b3f880215f315985fa3f6eca7c4d3
+  )
+  FetchContent_MakeAvailable(googletest)
+endif()
 
 set(TEST_TARGET_NAME "tests_continuous_batching")
 file(GLOB tests_src "*.cpp")

diff --git a/thirdparty/openvino_tokenizers b/thirdparty/openvino_tokenizers
+104 −45		src/CMakeLists.txt
+4 −0		src/case_fold.cpp
+4 −0		src/case_fold.hpp
+4 −0		src/normalize_unicode.cpp
+4 −0		src/normalize_unicode.hpp
+24 −9		src/ov_extension.cpp
+12 −2		src/tensorflow_translators.cpp
+8 −4		src/tensorflow_translators.hpp
+6 −3		src/tokenizer.hpp
+3 −0		src/wordpiece_tokenizer.cpp
+4 −0		src/wordpiece_tokenizer.hpp