llvm: upgrade to 18 (#1439)

* llvm: provide minimum changes needed for llvm 18 update

These changes make it possible to build against llvm 18.

In essence, there does not seem to be big problems. The only thing we
need to adjust is how we identify indirect calls done by way of VTables.
In essence we don't rely on types in function signatures from the LLVM
IR anymore as we want to use debug information for this anyways, so we
can kind of skip that part.

Signed-off-by: David Korczynski <david@adalogics.com>

* frontend: llvm: adjust patch

Signed-off-by: David Korczynski <david@adalogics.com>

* oss-fuzz: adjust patch

Signed-off-by: David Korczynski <david@adalogics.com>

* test: remove legacy pass manager since we do not use it anymore

Signed-off-by: David Korczynski <david@adalogics.com>

* oss-fuzz: add diff used for testing llvm18 in OSS-Fuzz

Signed-off-by: David Korczynski <david@adalogics.com>

---------

Signed-off-by: David Korczynski <david@adalogics.com>

frontends/llvm/lib/Transforms/FuzzIntrospector/FuzzIntrospector.cpp

@@ -909,9 +909,12 @@ std::string FuzzIntrospector::getFunctionFilename(Function *F) {
 std::string FuzzIntrospector::resolveTypeName(Type *T) {
   std::string RetType = "";
   std::string RetSuffix = "";
-  while (T->isPointerTy()) {
+  //while (T->isPointerTy()) {
+  if (T->isPointerTy()) {
     RetSuffix += "*";
+#if LLVM_VERSION_MAJOR < 16
     T = T->getPointerElementType();
+#endif
   }
   if (T->isIntegerTy()) {
     switch (T->getIntegerBitWidth()) {
@@ -997,6 +1000,7 @@ Function *FuzzIntrospector::value2Func(Value *Val) {
 
 // Recursively resolve a type and check if it is a function.
 bool FuzzIntrospector::isFunctionPointerType(Type *T) {
+#if LLVM_VERSION_MAJOR < 18
   if (PointerType *pointerType = dyn_cast<PointerType>(T)) {
 #if LLVM_VERSION_MAJOR >= 15
     if (!pointerType->isOpaque()) {
@@ -1007,6 +1011,7 @@ bool FuzzIntrospector::isFunctionPointerType(Type *T) {
     return isFunctionPointerType(pointerType->getPointerElementType());
 #endif
   }
+#endif
   return T->isFunctionTy();
 }
 
@@ -1099,6 +1104,10 @@ Function *FuzzIntrospector::extractVTableIndirectCall(Function *F,
     return nullptr;
   }
 
+#if LLVM_VERSION_MAJOR >= 18
+  return nullptr;
+#else
+
 #if LLVM_VERSION_MAJOR >= 15
   if (pointerType3->isOpaque()) {
     return nullptr;
@@ -1111,6 +1120,7 @@ Function *FuzzIntrospector::extractVTableIndirectCall(Function *F,
   Type *v13 = pointerType3->getPointerElementType();
 #endif
 
+
   if (!v13->isStructTy()) {
     return nullptr;
   }
@@ -1163,6 +1173,7 @@ Function *FuzzIntrospector::extractVTableIndirectCall(Function *F,
               VTableTargetFunc->getName().str().c_str());
   }
   return VTableTargetFunc;
+#endif
 }
 
 // Resolve all outgoing edges in a Function and populate

frontends/llvm/patch-llvm.sh

@@ -17,11 +17,9 @@
 set -x
 
 echo "add_subdirectory(FuzzIntrospector)" >> ./llvm/lib/Transforms/CMakeLists.txt
-sed -i 's/whole-program devirtualization and bitset lowering./whole-program devirtualization and bitset lowering.\nPM.add(createFuzzIntrospectorPass());/g' ./llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
-sed -i 's/using namespace/#include "llvm\/Transforms\/FuzzIntrospector\/FuzzIntrospector.h"\nusing namespace/g' ./llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
 sed -i 's/Instrumentation/Instrumentation\n  FuzzIntrospector/g' ./llvm/lib/Transforms/IPO/CMakeLists.txt
 
-sed -i 's/void initializeCrossDSOCFIPass(PassRegistry\&);/void initializeCrossDSOCFIPass(PassRegistry\&);\nvoid initializeFuzzIntrospectorPass(PassRegistry\&);/g' ./llvm/include/llvm/InitializePasses.h
+sed -i 's/void initializeXRayInstrumentationPass(PassRegistry\&);/void initializeXRayInstrumentationPass(PassRegistry\&);\nvoid initializeFuzzIntrospectorPass(PassRegistry\&);/g' ./llvm/include/llvm/InitializePasses.h
 sed -i 's/#include "llvm\/Transforms\/Instrumentation\/ThreadSanitizer.h"/#include "llvm\/Transforms\/Instrumentation\/ThreadSanitizer.h"\n#include "llvm\/Transforms\/FuzzIntrospector\/FuzzIntrospector.h"/g' ./llvm/lib/Passes/PassBuilder.cpp
 sed -i 's/#include "llvm\/Transforms\/Instrumentation\/PGOInstrumentation.h"/#include "llvm\/Transforms\/Instrumentation\/PGOInstrumentation.h"\n#include "llvm\/Transforms\/FuzzIntrospector\/FuzzIntrospector.h"/g' ./llvm/lib/Passes/PassBuilderPipelines.cpp
 sed -i 's/MPM.addPass(CrossDSOCFIPass());/MPM.addPass(CrossDSOCFIPass());\n  MPM.addPass(FuzzIntrospectorPass());/g' ./llvm/lib/Passes/PassBuilderPipelines.cpp

oss_fuzz_integration/oss-fuzz-patches.diff

@@ -1,5 +1,5 @@
 diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile
-index a9757a603..7515bbb7c 100644
+index 8dcbdce6c..583427556 100644
 --- a/infra/base-images/base-builder/Dockerfile
 +++ b/infra/base-images/base-builder/Dockerfile
 @@ -176,5 +176,9 @@ RUN chmod +x /usr/local/bin/clang-jcc && chmod +x /usr/local/bin/clang++-jcc
@@ -13,11 +13,73 @@ index a9757a603..7515bbb7c 100644
 
  CMD ["compile"]
 \ No newline at end of file
+diff --git a/infra/base-images/base-builder/compile b/infra/base-images/base-builder/compile
+index 04ac1cc84..6db4134b4 100755
+--- a/infra/base-images/base-builder/compile
++++ b/infra/base-images/base-builder/compile
+@@ -19,10 +19,10 @@ echo "---------------------------------------------------------------"
+
+ OSS_FUZZ_ON_DEMAND="${OSS_FUZZ_ON_DEMAND:-0}"
+
+-# This is a temporary fix: fall back to LLVM14's old pass manager
+-if [ -n "${OLD_LLVMPASS-}" ]; then
+-  export SANITIZER_FLAGS_introspector=$(echo $SANITIZER_FLAGS_introspector | sed -r 's/-O0/-flegacy-pass-manager/')
+-fi
++## This is a temporary fix: fall back to LLVM14's old pass manager
++#if [ -n "${OLD_LLVMPASS-}" ]; then
++#  export SANITIZER_FLAGS_introspector=$(echo $SANITIZER_FLAGS_introspector | sed -r 's/-O0/-flegacy-pass-manager/')
++#fi
+
+ if [ "$FUZZING_LANGUAGE" = "jvm" ]; then
+   if [ "$FUZZING_ENGINE" != "libfuzzer" ] && [ "$FUZZING_ENGINE" != "wycheproof" ]; then
+diff --git a/infra/base-images/base-builder/compile_libfuzzer b/infra/base-images/base-builder/compile_libfuzzer
+index 7962bd366..769bb8e73 100755
+--- a/infra/base-images/base-builder/compile_libfuzzer
++++ b/infra/base-images/base-builder/compile_libfuzzer
+@@ -21,6 +21,6 @@ if [ "$FUZZING_LANGUAGE" = "go" ]; then
+     export LIB_FUZZING_ENGINE="$LIB_FUZZING_ENGINE $GOPATH/gosigfuzz/gosigfuzz.o"
+ fi
+
+-cp /usr/local/lib/clang/*/lib/linux/libclang_rt.fuzzer-$ARCHITECTURE.a \
+-  $LIB_FUZZING_ENGINE_DEPRECATED
++#cp /usr/local/lib/clang/*/lib/linux/libclang_rt.fuzzer-$ARCHITECTURE.a \
++#  $LIB_FUZZING_ENGINE_DEPRECATED
+ echo " done."
+diff --git a/infra/base-images/base-builder/precompile_centipede b/infra/base-images/base-builder/precompile_centipede
+index 2abc1e9ff..63b90af50 100755
+--- a/infra/base-images/base-builder/precompile_centipede
++++ b/infra/base-images/base-builder/precompile_centipede
+@@ -16,7 +16,7 @@
+ ################################################################################
+
+ echo -n "Precompiling centipede"
+-
++exit 0
+ # Build Centipede with bazel.
+ cd "$SRC/fuzztest/centipede/"
+ apt-get update && apt-get install libssl-dev -y
+diff --git a/infra/base-images/base-builder/precompile_honggfuzz b/infra/base-images/base-builder/precompile_honggfuzz
+index df6bb2b75..dd7f3d1eb 100755
+--- a/infra/base-images/base-builder/precompile_honggfuzz
++++ b/infra/base-images/base-builder/precompile_honggfuzz
+@@ -28,6 +28,8 @@ PACKAGES=(
+
+ apt-get install -y ${PACKAGES[@]}
+
++exit 0
++
+ pushd $SRC/honggfuzz > /dev/null
+ make clean
+ # These CFLAGs match honggfuzz's default, with the exception of -mtune to
 diff --git a/infra/base-images/base-clang/Dockerfile b/infra/base-images/base-clang/Dockerfile
-index 7b22e0def..33dead533 100644
+index 0dd919813..155430185 100644
 --- a/infra/base-images/base-clang/Dockerfile
 +++ b/infra/base-images/base-clang/Dockerfile
-@@ -45,6 +45,8 @@ RUN apt-get update && apt-get install -y git && \
+@@ -42,9 +42,12 @@ RUN apt-get update && apt-get install -y git && \
+     apt-get autoremove --purge -y git && \
+     rm -rf .git
+
++COPY llvm-project /src/llvm-project
  COPY checkout_build_install_llvm.sh /root/
  # Keep all steps in the same script to decrease the number of intermediate
  # layes in docker file.
@@ -26,6 +88,89 @@ index 7b22e0def..33dead533 100644
  RUN /root/checkout_build_install_llvm.sh
  RUN rm /root/checkout_build_install_llvm.sh
 
+@@ -57,5 +60,6 @@ ENV CCC "clang++"
+ # https://llvm.org/docs/LibFuzzer.html#fuzzer-friendly-build-mode
+
+ ENV CFLAGS "-O1 -fno-omit-frame-pointer -gline-tables-only -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION"
+-ENV CXXFLAGS_EXTRA "-stdlib=libc++"
++#ENV CXXFLAGS_EXTRA "-stdlib=libc++"
++ENV CXXFLAGS_EXTRA ""
+ ENV CXXFLAGS "$CFLAGS $CXXFLAGS_EXTRA"
+diff --git a/infra/base-images/base-clang/checkout_build_install_llvm.sh b/infra/base-images/base-clang/checkout_build_install_llvm.sh
+index 65f0ea554..549a190f6 100755
+--- a/infra/base-images/base-clang/checkout_build_install_llvm.sh
++++ b/infra/base-images/base-clang/checkout_build_install_llvm.sh
+@@ -50,14 +50,16 @@ LLVM_DEP_PACKAGES="build-essential make ninja-build git python3 python3-distutil
+ apt-get update && apt-get install -y $LLVM_DEP_PACKAGES --no-install-recommends
+
+ # For manual bumping.
+-OUR_LLVM_REVISION=llvmorg-15-init-1464-gbf7f8d6f
++#OUR_LLVM_REVISION=llvmorg-15-init-1464-gbf7f8d6f
++OUR_LLVM_REVISION=llvmorg-18-init-14420-gea3a3b25
+
+ mkdir $SRC/chromium_tools
+ cd $SRC/chromium_tools
+ git clone https://chromium.googlesource.com/chromium/src/tools/clang
+ cd clang
+ # Pin clang due to https://github.com/google/oss-fuzz/issues/7617
+-git checkout 946a41a51f44207941b3729a0733dfc1e236644e
++#git checkout 946a41a51f44207941b3729a0733dfc1e236644e
++git checkout 9eb79319239629c1b23cf7a59e5ebb2bab319a34
+
+ # To allow for manual downgrades. Set to 0 to use Chrome's clang version (i.e.
+ # *not* force a manual downgrade). Set to 1 to force a manual downgrade.
+@@ -89,15 +91,18 @@ function clone_with_retries {
+   set -e
+   return $CHECKOUT_RETURN_CODE
+ }
+-clone_with_retries https://github.com/llvm/llvm-project.git $LLVM_SRC
++#COPY llvm-project $LLVM_SRC
++#clone_with_retries https://github.com/llvm/llvm-project.git $LLVM_SRC
+
+-PROJECTS_TO_BUILD="libcxx;libcxxabi;compiler-rt;clang;lld"
++#PROJECTS_TO_BUILD="libcxx;libcxxabi;compiler-rt;clang;lld"
++PROJECTS_TO_BUILD="compiler-rt;clang;lld"
+ function cmake_llvm {
+   extra_args="$@"
+   cmake -G "Ninja" \
+       -DLIBCXX_ENABLE_SHARED=OFF \
+       -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \
+       -DLIBCXXABI_ENABLE_SHARED=OFF \
++      -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
+       -DCMAKE_BUILD_TYPE=Release \
+       -DLLVM_TARGETS_TO_BUILD="$TARGET_TO_BUILD" \
+       -DLLVM_ENABLE_PROJECTS="$PROJECTS_TO_BUILD" \
+@@ -213,20 +218,21 @@ then
+   # do not support MSAN nor do we care about i386.
+   exit 0
+ fi
+-
+-function cmake_libcxx {
+-  extra_args="$@"
+-  cmake -G "Ninja" \
+-      -DLIBCXX_ENABLE_SHARED=OFF \
+-      -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \
+-      -DLIBCXXABI_ENABLE_SHARED=OFF \
++free_disk_space
++exit 0
++#function cmake_libcxx {
++#  extra_args="$@"
++#  cmake -G "Ninja" \
++#      -DLIBCXX_ENABLE_SHARED=OFF \
++#      -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \
++#      -DLIBCXXABI_ENABLE_SHARED=OFF \
+       -DCMAKE_BUILD_TYPE=Release \
+       -DLLVM_TARGETS_TO_BUILD="$TARGET_TO_BUILD" \
+-      -DLLVM_ENABLE_PROJECTS="libcxx;libcxxabi" \
++#      -DLLVM_ENABLE_PROJECTS="libcxx;libcxxabi" \
+       -DLLVM_BINUTILS_INCDIR="/usr/include/" \
+       $extra_args \
+       $LLVM_SRC/llvm
+-}
++#}
+
+ # 32-bit libraries.
+ mkdir -p $WORK/i386
 diff --git a/infra/base-images/base-runner/Dockerfile b/infra/base-images/base-runner/Dockerfile
 index 45c5e73b6..07bd1b9c0 100755
 --- a/infra/base-images/base-runner/Dockerfile
@@ -48,3 +193,16 @@ index 45c5e73b6..07bd1b9c0 100755
 
  # Copy the binaries needed for code coverage and crash symbolization.
  COPY --from=base-clang /usr/local/bin/llvm-cov \
+diff --git a/projects/leveldb/fuzz_db.cc b/projects/leveldb/fuzz_db.cc
+index 0147c124f..5cb9f166a 100644
+--- a/projects/leveldb/fuzz_db.cc
++++ b/projects/leveldb/fuzz_db.cc
+@@ -40,7 +40,7 @@ class AutoDbDeleter {
+   AutoDbDeleter& operator=(const AutoDbDeleter&) = delete;
+
+   ~AutoDbDeleter() {
+-    std::__fs::filesystem::remove_all(kDbPath);
++    //std::__fs::filesystem::remove_all(kDbPath);
+   }
+ };
+

tests/cpp-simple-example-1/build_all.sh

@@ -23,5 +23,5 @@ cd work
 echo "[+] Linking the projects"
 ../../../build/llvm-build/bin/clang++ -v -fsanitize=fuzzer-no-link -g -c -flto  ../fuzzer.cpp -o fuzzer.o
 echo "dos"
-../../../build/llvm-build/bin/clang++ -v -fsanitize=fuzzer -g -flto  -flegacy-pass-manager  fuzzer.o -o fuzzer
+../../../build/llvm-build/bin/clang++ -v -fsanitize=fuzzer -g -flto  fuzzer.o -o fuzzer
 echo "dres"

tests/simple-example-0/fuzzer.c

@@ -1,6 +1,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdint.h>
+#include <string.h>
 
 int unreached_target2(const uint8_t *data) {
         return 5;
@@ -50,7 +51,7 @@ int unreached_target3(const uint8_t *data, size_t *theval) {
     return 5;
 }
 
-char *d = {0x12};
+char *d = "sf";
 int target2(const uint8_t *data) {
 	if (data[0] == 0x41) return 1;
     unreached_target1(d);