Merge branch 'souffle-2.3' into 'main'

Upgrade to Souffle 2.3 Closes #404 See merge request rewriting/ddisasm!865
GrammaTech · Jul 6, 2022 · 5953177 · 5953177
2 parents 2be3e6e + 95da138
commit 5953177
Show file tree

Hide file tree

Showing 8 changed files with 72 additions and 113 deletions.
diff --git a/.ci/Dockerfile.static b/.ci/Dockerfile.static
@@ -18,10 +18,11 @@ RUN apt-get -y update \
       libtool \
       lsb-release \
       mcpp \
+      python \
       sqlite3 \
       zlib1g-dev
 
-RUN git clone -b 2.2 https://github.com/souffle-lang/souffle && \
+RUN git clone -b 2.3 https://github.com/souffle-lang/souffle && \
     cd souffle && \
     cmake . -Bbuild -DCMAKE_BUILD_TYPE=Release -DSOUFFLE_USE_CURSES=0 -DSOUFFLE_USE_SQLITE=0 -DSOUFFLE_DOMAIN_64BIT=1 && \
     make -C build install -j4
@@ -104,9 +105,7 @@ RUN ln -s /usr/i686-linux-gnu/lib/ /usr/lib/i386-linux-gnu && \
     mkdir etc && \
     ldconfig -c etc/ld.do.cache -r .
 
-COPY --from=souffle /usr/local/bin/souffle-compile /usr/bin/souffle-compile
-COPY --from=souffle /usr/local/bin/souffle /usr/bin/souffle
-COPY --from=souffle /usr/local/bin/souffleprof /usr/bin/souffleprof
+COPY --from=souffle /usr/local/bin/souffle* /usr/local/bin/
 COPY --from=souffle /usr/local/include/souffle/ /usr/include/souffle
 COPY --from=lief /usr/lib/libLIEF.a /usr/lib/libLIEF.a
 COPY --from=lief /usr/include/LIEF /usr/include/LIEF

diff --git a/.ci/Dockerfile.ubuntu20 b/.ci/Dockerfile.ubuntu20
@@ -18,10 +18,11 @@ RUN apt-get -y update \
       libtool \
       lsb-release \
       mcpp \
+      python \
       sqlite3 \
       zlib1g-dev
 
-RUN git clone -b 2.2 https://github.com/souffle-lang/souffle && \
+RUN git clone -b 2.3 https://github.com/souffle-lang/souffle && \
     cd souffle && \
     cmake . -Bbuild -DCMAKE_BUILD_TYPE=Release -DSOUFFLE_USE_CURSES=0 -DSOUFFLE_USE_SQLITE=0 -DSOUFFLE_DOMAIN_64BIT=1 && \
     make -C build install -j4

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -61,8 +61,3 @@ SouffleProf
 Generating HTML files...
 file output to: profiler_html/1.html
 ```
-
-The current version of Souffle (2.2) has a bug where synthesized profiled
-programs are missing an `#include` for `Logger.h`; see
-https://github.com/souffle-lang/souffle/pull/2186. The easiest way to run
-profiling is to install Souffle from the commit that fixes this bug, 1f2b7c8.
diff --git a/Dockerfile b/Dockerfile
@@ -21,7 +21,7 @@ RUN apt-get -y update \
       sqlite3 \
       zlib1g-dev
 
-RUN git clone -b 2.2 https://github.com/souffle-lang/souffle && \
+RUN git clone -b 2.3 https://github.com/souffle-lang/souffle && \
     cd souffle && \
     cmake . -Bbuild -DCMAKE_BUILD_TYPE=Release -DSOUFFLE_USE_CURSES=0 -DSOUFFLE_USE_SQLITE=0 -DSOUFFLE_DOMAIN_64BIT=1 && \
     cd build && \

diff --git a/README.md b/README.md
@@ -54,7 +54,7 @@ To build ddisasm from source, the following requirements should be installed:
 - [Capstone](http://www.capstone-engine.org/), version 5.0.0 or later
   - 5.x is not yet released by the Capstone team.
   - GrammaTech builds and tests using the [GrammaTech/capstone](https://github.com/GrammaTech/capstone) fork.
-- [Souffle](https://souffle-lang.github.io), version 2.2
+- [Souffle](https://souffle-lang.github.io), version 2.3
   - Must be configured with support for 64 bit numbers (via `-DSOUFFLE_DOMAIN_64BIT=1` during configuration)
 - [libehp](https://git.zephyr-software.com/opensrc/libehp), version 1.0.0 or higher
 - [LIEF](https://lief.quarkslab.com/), version 0.12.1 or higher

diff --git a/conanfile.py b/conanfile.py
@@ -76,20 +76,16 @@ class DdisasmConan(Properties, ConanFile):
 
     lief_version = "0.12.1"
     libehp_version = "0.1.1-gt3"
-    souffle_version = "2.2"
+    souffle_version = "2.3"
     build_requires = (
         "libehp/%s@rewriting+extra-packages/stable" % (libehp_version),
         "lief/%s@rewriting+extra-packages/stable" % (lief_version),
+        "souffle/%s@rewriting+extra-packages/stable" % (souffle_version),
     )
 
     def build_requirements(self):
         if self.settings.os == "Windows":
             self.build_requires("ninja/1.10.2")
-        else:
-            self.build_requires(
-                "souffle/%s@rewriting+extra-packages/stable"
-                % (self.souffle_version)
-            )
 
     boost_version = "1.69.0"
     gtirb_version = "dev"
@@ -128,16 +124,20 @@ def build(self):
             self.build_cmake()
 
     # Puts a dependency's bin path on PATH
-    def add_dep_bin_path(self, dep):
-        bin_dirs = self.deps_cpp_info[dep].bin_paths
+    def add_dep_bin_path(self, *deps):
+        bin_dirs = sum([self.deps_cpp_info[dep].bin_paths for dep in deps], [])
         new_path = [os.environ.get("PATH", "")] + bin_dirs
         os.environ["PATH"] = os.pathsep.join(new_path)
 
-    # Puts a dependency's lib path on LD_LIBRARY_PATH
+    # Puts a dependency's lib path on LD_LIBRARY_PATH (for Linux) or PATH (for
+    # Windows)
     def add_dep_lib_path(self, *deps):
         lib_dirs = sum([self.deps_cpp_info[dep].lib_paths for dep in deps], [])
-        new_ld_lib_path = [os.environ.get("LD_LIBRARY_PATH", "")] + lib_dirs
-        os.environ["LD_LIBRARY_PATH"] = os.pathsep.join(new_ld_lib_path)
+        env_var_name = (
+            "PATH" if self.settings.os == "Windows" else "LD_LIBRARY_PATH"
+        )
+        new_value = [os.environ.get(env_var_name, "")] + lib_dirs
+        os.environ[env_var_name] = os.pathsep.join(new_value)
 
     def build_cmake(self):
         defs = {"CMAKE_VERBOSE_MAKEFILE:BOOL": "ON", "ENABLE_CONAN:BOOL": "ON"}
@@ -148,7 +148,6 @@ def build_cmake(self):
                     k: os.environ.get(k)
                     for k in [
                         "CMAKE_PREFIX_PATH",
-                        "SOUFFLE_INCLUDE_DIR",
                         "PYTHON",
                     ]
                 }
@@ -157,18 +156,18 @@ def build_cmake(self):
             defs[
                 "CMAKE_CXX_FLAGS"
             ] = "/DBOOST_ALL_NO_LIB /DBOOST_UUID_FORCE_AUTO_LINK"
+            self.add_dep_lib_path("libffi")
         else:
             cmake = CMake(self, generator=None, parallel=True)
             defs.update({"GTIRB_PPRINTER_STRIP_DEBUG_SYMBOLS:BOOL": "ON"})
-            self.add_dep_bin_path("mcpp")
 
         revision = os.environ.get("CI_COMMIT_SHORT_SHA")
         if revision:
             defs["DDISASM_BUILD_REVISION"] = revision
 
         if self.settings.build_type == "Release":
             cmake.build_type = "RelWithDebInfo"
-        self.add_dep_bin_path("gtirb-pprinter")
+        self.add_dep_bin_path("gtirb-pprinter", "mcpp")
         self.add_dep_lib_path("gtirb-pprinter", "gtirb", "capstone")
         bin_dir = os.path.join(os.getcwd(), "bin")
         os.environ["PATH"] = os.pathsep.join([os.environ["PATH"], bin_dir])

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -1,26 +1,28 @@
 # ===== generated souffle code =====
-# On windows, wsl is used to call souffle.  There is no good way to 'find'
-# souffle in wsl, so we skip this step entirely and will hard-code the call.
-if(NOT WIN32)
-  find_program(
-    SOUFFLE souffle
-    HINTS $ENV{PATH}
-    DOC "souffle must be in your PATH to build disassembler.")
-  if(NOT SOUFFLE)
-    message(FATAL_ERROR "souffle was not found in your PATH. Unable to build.")
-  endif()
-  if(NOT SOUFFLE_INCLUDE_DIR)
-    get_filename_component(SOUFFLE_BIN_DIR ${SOUFFLE} DIRECTORY)
-    get_filename_component(SOUFFLE_INCLUDE_DIR ${SOUFFLE_BIN_DIR}/../include
-                           ABSOLUTE)
-    if(NOT EXISTS ${SOUFFLE_INCLUDE_DIR}/souffle)
-      message(WARNING "SOUFFLE_INCLUDE_DIR not found")
-      unset(SOUFFLE_INCLUDE_DIR)
-    endif()
-    unset(SOUFFLE_BIN_DIR)
+find_program(
+  SOUFFLE souffle
+  HINTS $ENV{PATH}
+  DOC "souffle must be in your PATH to build disassembler.")
+if(NOT SOUFFLE)
+  message(FATAL_ERROR "souffle was not found in your PATH. Unable to build.")
+endif()
+if(NOT SOUFFLE_INCLUDE_DIR)
+  get_filename_component(SOUFFLE_BIN_DIR ${SOUFFLE} DIRECTORY)
+  get_filename_component(SOUFFLE_INCLUDE_DIR ${SOUFFLE_BIN_DIR}/../include
+                         ABSOLUTE)
+  if(NOT EXISTS ${SOUFFLE_INCLUDE_DIR}/souffle)
+    message(WARNING "SOUFFLE_INCLUDE_DIR not found")
+    unset(SOUFFLE_INCLUDE_DIR)
   endif()
+  unset(SOUFFLE_BIN_DIR)
+endif()
+
+# Souffle searches for a "mcpp" binary, even on Windows, where "mcpp.exe" would
+# probably make more sense. We explicitly provide the preprocessor.
+if(WIN32)
+  set(SOUFFLE_PREPROCESSOR_ARG "--preprocessor=mcpp.exe")
 else()
-  set(SOUFFLE wsl souffle)
+  set(SOUFFLE_PREPROCESSOR_ARG "")
 endif()
 
 set(DATALOG_BASE_SOURCES
@@ -67,23 +69,17 @@ if(DDISASM_ARM_32)
       datalog/arch/arm32/memory_access.dl)
 
   set(GENERATED_ARM32_CPP souffle_disasm_arm32.cpp)
-
-  if(WIN32)
-    set(GENERATED_ARM32_CPP_PATH
-        "$$(wslpath ${CMAKE_BINARY_DIR}/src/souffle_disasm_arm32.cpp)")
-  else()
-    set(GENERATED_ARM32_CPP_PATH
-        ${CMAKE_BINARY_DIR}/src/souffle_disasm_arm32.cpp)
-  endif()
+  set(GENERATED_ARM32_CPP_PATH ${CMAKE_BINARY_DIR}/src/souffle_disasm_arm32.cpp)
 
   add_custom_command(
     OUTPUT ${GENERATED_ARM32_CPP}
            # Souffle includes the path of the output file in the generated
            # program name. Change directory and use a relative path so the name
            # does not depend on build location.
     WORKING_DIRECTORY "${SOUFFLE_DATALOG_DIR}"
-    COMMAND ${SOUFFLE} main.dl -g ${GENERATED_ARM32_CPP_PATH} -jauto
-            -MARCH_ARM32
+    COMMAND
+      ${SOUFFLE} main.dl -g ${GENERATED_ARM32_CPP_PATH} -jauto -MARCH_ARM32
+      ${SOUFFLE_PREPROCESSOR_ARG}
     DEPENDS ${DATALOG_BASE_SOURCES} ${DATALOG_ARM32_SOURCES})
 endif()
 
@@ -100,23 +96,18 @@ if(DDISASM_ARM_64)
       datalog/arch/arm64/memory_access.dl)
 
   set(GENERATED_ARM64_CPP souffle_disasm_arm64.cpp)
-
-  if(WIN32)
-    set(GENERATED_ARM64_CPP_PATH
-        "$$(wslpath ${CMAKE_BINARY_DIR}/src/souffle_disasm_arm64.cpp)")
-  else()
-    set(GENERATED_ARM64_CPP_PATH
-        "${CMAKE_BINARY_DIR}/src/souffle_disasm_arm64.cpp")
-  endif()
+  set(GENERATED_ARM64_CPP_PATH
+      "${CMAKE_BINARY_DIR}/src/souffle_disasm_arm64.cpp")
 
   add_custom_command(
     OUTPUT ${GENERATED_ARM64_CPP}
            # Souffle includes the path of the output file in the generated
            # program name. Change directory and use a relative path so the name
            # does not depend on build location.
     WORKING_DIRECTORY "${SOUFFLE_DATALOG_DIR}"
-    COMMAND ${SOUFFLE} main.dl -g ${GENERATED_ARM64_CPP_PATH} -jauto
-            -MARCH_ARM64
+    COMMAND
+      ${SOUFFLE} main.dl -g ${GENERATED_ARM64_CPP_PATH} -jauto -MARCH_ARM64
+      ${SOUFFLE_PREPROCESSOR_ARG}
     DEPENDS ${DATALOG_BASE_SOURCES} ${DATALOG_ARM64_SOURCES})
 endif()
 
@@ -133,22 +124,18 @@ if(DDISASM_MIPS_32)
 
   set(GENERATED_MIPS32_CPP souffle_disasm_mips32.cpp)
 
-  if(WIN32)
-    set(GENERATED_MIPS32_CPP_PATH
-        "$$(wslpath ${CMAKE_BINARY_DIR}/src/souffle_disasm_mips32.cpp)")
-  else()
-    set(GENERATED_MIPS32_CPP_PATH
-        "${CMAKE_BINARY_DIR}/src/souffle_disasm_mips32.cpp")
-  endif()
+  set(GENERATED_MIPS32_CPP_PATH
+      "${CMAKE_BINARY_DIR}/src/souffle_disasm_mips32.cpp")
 
   add_custom_command(
     OUTPUT ${GENERATED_MIPS32_CPP}
            # Souffle includes the path of the output file in the generated
            # program name. Change directory and use a relative path so the name
            # does not depend on build location.
     WORKING_DIRECTORY "${SOUFFLE_DATALOG_DIR}"
-    COMMAND ${SOUFFLE} main.dl -g ${GENERATED_MIPS32_CPP_PATH} -jauto
-            -MARCH_MIPS32
+    COMMAND
+      ${SOUFFLE} main.dl -g ${GENERATED_MIPS32_CPP_PATH} -jauto -MARCH_MIPS32
+      ${SOUFFLE_PREPROCESSOR_ARG}
     DEPENDS ${DATALOG_BASE_SOURCES} ${DATALOG_MIPS32_SOURCES})
 endif()
 
@@ -164,23 +151,18 @@ if(DDISASM_X86_32)
       datalog/arch/intel/memory_access.dl)
 
   set(GENERATED_X86_32_CPP souffle_disasm_x86_32.cpp)
-
-  if(WIN32)
-    set(GENERATED_X86_32_CPP_PATH
-        "$$(wslpath ${CMAKE_BINARY_DIR}/src/souffle_disasm_x86_32.cpp)")
-  else()
-    set(GENERATED_X86_32_CPP_PATH
-        "${CMAKE_BINARY_DIR}/src/souffle_disasm_x86_32.cpp")
-  endif()
+  set(GENERATED_X86_32_CPP_PATH
+      "${CMAKE_BINARY_DIR}/src/souffle_disasm_x86_32.cpp")
 
   add_custom_command(
     OUTPUT ${GENERATED_X86_32_CPP}
            # Souffle includes the path of the output file in the generated
            # program name. Change directory and use a relative path so the name
            # does not depend on build location.
     WORKING_DIRECTORY "${SOUFFLE_DATALOG_DIR}"
-    COMMAND ${SOUFFLE} main.dl -g ${GENERATED_X86_32_CPP_PATH} -jauto
-            -MARCH_IA32
+    COMMAND
+      ${SOUFFLE} main.dl -g ${GENERATED_X86_32_CPP_PATH} -jauto -MARCH_IA32
+      ${SOUFFLE_PREPROCESSOR_ARG}
     DEPENDS ${DATALOG_BASE_SOURCES} ${DATALOG_X86_32_SOURCES})
 endif()
 
@@ -196,23 +178,18 @@ if(DDISASM_X86_64)
       datalog/arch/intel/memory_access.dl)
 
   set(GENERATED_X86_64_CPP souffle_disasm_x86_64.cpp)
-
-  if(WIN32)
-    set(GENERATED_X86_64_CPP_PATH
-        "$$(wslpath ${CMAKE_BINARY_DIR}/src/souffle_disasm_x86_64.cpp)")
-  else()
-    set(GENERATED_X86_64_CPP_PATH
-        "${CMAKE_BINARY_DIR}/src/souffle_disasm_x86_64.cpp")
-  endif()
+  set(GENERATED_X86_64_CPP_PATH
+      "${CMAKE_BINARY_DIR}/src/souffle_disasm_x86_64.cpp")
 
   add_custom_command(
     OUTPUT ${GENERATED_X86_64_CPP}
            # Souffle includes the path of the output file in the generated
            # program name. Change directory and use a relative path so the name
            # does not depend on build location.
     WORKING_DIRECTORY "${SOUFFLE_DATALOG_DIR}"
-    COMMAND ${SOUFFLE} main.dl -g ${GENERATED_X86_64_CPP_PATH} -jauto
-            -MARCH_AMD64
+    COMMAND
+      ${SOUFFLE} main.dl -g ${GENERATED_X86_64_CPP_PATH} -jauto -MARCH_AMD64
+      ${SOUFFLE_PREPROCESSOR_ARG}
     DEPENDS ${DATALOG_BASE_SOURCES} ${DATALOG_X86_64_SOURCES})
 endif()
 

diff --git a/src/passes/CMakeLists.txt b/src/passes/CMakeLists.txt
@@ -19,14 +19,8 @@ endif()
 # ============ No return pass =================
 
 set(NO_RETURN_DATALOG_SOURCES datalog/no_return_analysis.dl)
-
-if(WIN32)
-  set(NO_RETURN_DATALOG_MAIN
-      "$$(wslpath ${CMAKE_CURRENT_SOURCE_DIR}/datalog/no_return_analysis.dl)")
-else()
-  set(NO_RETURN_DATALOG_MAIN
-      ${CMAKE_CURRENT_SOURCE_DIR}/datalog/no_return_analysis.dl)
-endif()
+set(NO_RETURN_DATALOG_MAIN
+    ${CMAKE_CURRENT_SOURCE_DIR}/datalog/no_return_analysis.dl)
 
 set(NO_RETURN_CPP "${CMAKE_BINARY_DIR}/src/passes/souffle_no_return.cpp")
 
@@ -37,6 +31,7 @@ add_custom_command(
          # depend on build location.
   WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/src/passes"
   COMMAND ${SOUFFLE} ${NO_RETURN_DATALOG_MAIN} -g souffle_no_return.cpp -jauto
+          ${SOUFFLE_PREPROCESSOR_ARG}
   DEPENDS ${NO_RETURN_DATALOG_SOURCES})
 
 add_library(no_return_pass STATIC NoReturnPass.cpp ${NO_RETURN_CPP})
@@ -61,16 +56,9 @@ endif()
 
 # ============ Function inference pass =========
 
-if(WIN32)
-  set(FUNCTION_INFERENCE_DATALOG_MAIN
-      "$$(wslpath ${CMAKE_CURRENT_SOURCE_DIR}/datalog/function_inference.dl)")
-else()
-  set(FUNCTION_INFERENCE_DATALOG_MAIN
-      ${CMAKE_CURRENT_SOURCE_DIR}/datalog/function_inference.dl)
-endif()
-
+set(FUNCTION_INFERENCE_DATALOG_MAIN
+    ${CMAKE_CURRENT_SOURCE_DIR}/datalog/function_inference.dl)
 set(FUNCTION_INFERENCE_DATALOG_SOURCES datalog/function_inference.dl)
-
 set(FUNCTION_INFERENCE_CPP
     "${CMAKE_BINARY_DIR}/src/passes/souffle_function_inference.cpp")
 
@@ -81,7 +69,7 @@ add_custom_command(
          # depend on build location.
   WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/src/passes"
   COMMAND ${SOUFFLE} ${FUNCTION_INFERENCE_DATALOG_MAIN} -g
-          souffle_function_inference.cpp -jauto
+          souffle_function_inference.cpp -jauto ${SOUFFLE_PREPROCESSOR_ARG}
   DEPENDS ${FUNCTION_INFERENCE_DATALOG_SOURCES})
 
 add_library(function_inference_pass STATIC FunctionInferencePass.cpp