Merge branch 'bugfix/reproducible_builds_improvements' into 'master'

build system: reproducible build improvements

Closes IDFGH-12690

See merge request espressif/esp-idf!32734

.gitlab/ci/host-test.yml

@@ -68,22 +68,6 @@ test_ldgen_on_host:
   variables:
     LC_ALL: C.UTF-8
 
-test_reproducible_build:
-  extends: .host_test_template
-  script:
-    - ./tools/ci/test_reproducible_build.sh
-  artifacts:
-    when: on_failure
-    paths:
-      - "**/sdkconfig"
-      - "**/build*/*.bin"
-      - "**/build*/*.elf"
-      - "**/build*/*.map"
-      - "**/build*/flasher_args.json"
-      - "**/build*/*.bin"
-      - "**/build*/bootloader/*.bin"
-      - "**/build*/partition_table/*.bin"
-
 test_spiffs_on_host:
   extends: .host_test_template
   script:

.gitlab/ci/rules.yml

@@ -108,8 +108,6 @@
   - "tools/detect_python.sh"
   - "tools/detect_python.fish"
 
-  - "tools/ci/test_reproducible_build.sh"
-
   - "tools/gen_soc_caps_kconfig/*"
   - "tools/gen_soc_caps_kconfig/test/test_gen_soc_caps_kconfig.py"
 

CMakeLists.txt

@@ -152,46 +152,8 @@ if(CONFIG_COMPILER_DUMP_RTL_FILES)
     list(APPEND compile_options "-fdump-rtl-expand")
 endif()
 
-if(NOT ${CMAKE_C_COMPILER_VERSION} VERSION_LESS 8.0.0)
-    if(CONFIG_COMPILER_HIDE_PATHS_MACROS)
-        list(APPEND compile_options "-fmacro-prefix-map=${CMAKE_SOURCE_DIR}=.")
-        list(APPEND compile_options "-fmacro-prefix-map=${IDF_PATH}=/IDF")
-    endif()
-
-    if(CONFIG_APP_REPRODUCIBLE_BUILD)
-        idf_build_set_property(DEBUG_PREFIX_MAP_GDBINIT "${BUILD_DIR}/prefix_map_gdbinit")
-
-        list(APPEND compile_options "-fdebug-prefix-map=${IDF_PATH}=/IDF")
-        list(APPEND compile_options "-fdebug-prefix-map=${PROJECT_DIR}=/IDF_PROJECT")
-        list(APPEND compile_options "-fdebug-prefix-map=${BUILD_DIR}=/IDF_BUILD")
-
-        # component dirs
-        idf_build_get_property(python PYTHON)
-        idf_build_get_property(idf_path IDF_PATH)
-        idf_build_get_property(component_dirs BUILD_COMPONENT_DIRS)
-
-        execute_process(
-            COMMAND ${python}
-                "${idf_path}/tools/generate_debug_prefix_map.py"
-                "${BUILD_DIR}"
-                "${component_dirs}"
-            OUTPUT_VARIABLE result
-            RESULT_VARIABLE ret
-        )
-        if(NOT ret EQUAL 0)
-            message(FATAL_ERROR "This is a bug. Please report to https://github.com/espressif/esp-idf/issues")
-        endif()
-
-        spaces2list(result)
-        list(LENGTH component_dirs length)
-        math(EXPR max_index "${length} - 1")
-        foreach(index RANGE ${max_index})
-            list(GET component_dirs ${index} folder)
-            list(GET result ${index} after)
-            list(APPEND compile_options "-fdebug-prefix-map=${folder}=${after}")
-        endforeach()
-    endif()
-endif()
+__generate_prefix_map(prefix_map_compile_options)
+list(APPEND compile_options ${prefix_map_compile_options})
 
 if(CONFIG_COMPILER_DISABLE_GCC12_WARNINGS)
     list(APPEND compile_options "-Wno-address"

docs/en/api-guides/reproducible-builds.rst

@@ -13,6 +13,7 @@ When reproducible builds are enabled, the application built with ESP-IDF does no
 - Directory where the project is located
 - Directory where ESP-IDF is located (``IDF_PATH``)
 - Build time
+- Toolchain installation path
 
 Reasons for Non-Reproducible Builds
 -----------------------------------
@@ -46,6 +47,7 @@ ESP-IDF achieves reproducible builds using the following measures:
     - Path to the project is replaced with ``/IDF_PROJECT``
     - Path to the build directory is replaced with ``/IDF_BUILD``
     - Paths to components are replaced with ``/COMPONENT_NAME_DIR`` (where ``NAME`` is the name of the component)
+    - Path to the toolchain is replaced with ``/TOOLCHAIN``
 
 - Build date and time are not included into the :ref:`application  metadata structure <app-image-format-application-description>` and :ref:`bootloader metadata structure <image-format-bootloader-description>` if :ref:`CONFIG_APP_REPRODUCIBLE_BUILD` is enabled.
 - ESP-IDF build system ensures that source file lists, component lists and other sequences are sorted before passing them to CMake. Various other parts of the build system, such as the linker script generator also perform sorting to ensure that same output is produced regardless of the environment.

docs/zh_CN/api-guides/reproducible-builds.rst

@@ -13,6 +13,7 @@ ESP-IDF 构建系统支持 `可重复构建 <https://reproducible-builds.org/doc
 - 项目所在目录
 - ESP-IDF 所在目录 (``IDF_PATH``)
 - 构建时间
+- 工具链安装路径
 
 构建不可重复的原因
 ------------------
@@ -46,6 +47,7 @@ ESP-IDF 可通过以下方式实现可重复构建：
     - 替换项目路径为 ``/IDF_PROJECT``
     - 替换构建目录的路径为 ``/IDF_BUILD``
     - 替换组件路径为 ``/COMPONENT_NAME_DIR`` （其中 ``NAME`` 指的是组件的名称）
+    - 替换工具链的路径为 ``/TOOLCHAIN``
 
 - 如果启用 :ref:`CONFIG_APP_REPRODUCIBLE_BUILD`，则不会将构建日期和时间包括在 :ref:`应用程序元数据结构 <app-image-format-application-description>` 和 :ref:`引导加载程序元数据结构 <image-format-bootloader-description>` 中。
 - ESP-IDF 构建系统在将源文件列表、组件列表和其他序列传递给 CMake 之前会对其进行排序。构建系统的其他各个部分，如链接器脚本生成器，也会先排序，从而确保无论环境如何，输出都一致。

tools/ci/exclude_check_tools_files.txt

@@ -8,7 +8,6 @@ tools/ci/get_all_test_results.py
 tools/gdb_panic_server.py
 tools/check_term.py
 tools/python_version_checker.py
-tools/generate_debug_prefix_map.py
 tools/ci/astyle-rules.yml
 tools/ci/checkout_project_ref.py
 tools/ci/ci_fetch_submodule.py

tools/ci/executable-list.txt

@@ -80,7 +80,6 @@ tools/ci/push_to_github.sh
 tools/ci/sort_yaml.py
 tools/ci/test_autocomplete/test_autocomplete.py
 tools/ci/test_configure_ci_environment.sh
-tools/ci/test_reproducible_build.sh
 tools/docker/entrypoint.sh
 tools/esp_app_trace/logtrace_proc.py
 tools/esp_app_trace/sysviewtrace_proc.py

tools/cmake/idf.cmake

@@ -48,6 +48,7 @@ if(NOT __idf_env_set)
     include(ldgen)
     include(dfu)
     include(version)
+    include(prefix_map)
 
     __build_init("${idf_path}")
 

tools/cmake/prefix_map.cmake

@@ -0,0 +1,54 @@
+# Utilities for remapping path prefixes
+#
+# __generate_prefix_map
+# Prepares the list of compiler flags for remapping various paths
+# to fixed names. This is used when reproducible builds are required.
+# This function also creates a gdbinit file for the debugger to
+# remap the substituted paths back to the real paths in the filesystem.
+function(__generate_prefix_map compile_options_var)
+    set(compile_options)
+    idf_build_get_property(idf_path IDF_PATH)
+    idf_build_get_property(build_components BUILD_COMPONENTS)
+
+    if(CONFIG_COMPILER_HIDE_PATHS_MACROS)
+        list(APPEND compile_options "-fmacro-prefix-map=${CMAKE_SOURCE_DIR}=.")
+        list(APPEND compile_options "-fmacro-prefix-map=${idf_path}=/IDF")
+    endif()
+
+    if(CONFIG_APP_REPRODUCIBLE_BUILD)
+        list(APPEND compile_options "-fdebug-prefix-map=${idf_path}=/IDF")
+        list(APPEND compile_options "-fdebug-prefix-map=${PROJECT_DIR}=/IDF_PROJECT")
+        list(APPEND compile_options "-fdebug-prefix-map=${BUILD_DIR}=/IDF_BUILD")
+
+        # Generate mapping for component paths
+        set(gdbinit_file_lines)
+        foreach(component_name ${build_components})
+            idf_component_get_property(component_dir ${component_name} COMPONENT_DIR)
+
+            string(TOUPPER ${component_name} component_name_uppercase)
+            set(substituted_path "/COMPONENT_${component_name_uppercase}_DIR")
+            list(APPEND compile_options "-fdebug-prefix-map=${component_dir}=${substituted_path}")
+            string(APPEND gdbinit_file_lines "set substitute-path ${substituted_path} ${component_dir}\n")
+        endforeach()
+
+        # Mapping for toolchain path
+        execute_process(
+            COMMAND ${CMAKE_C_COMPILER} -print-sysroot
+            OUTPUT_VARIABLE compiler_sysroot
+        )
+        if(compiler_sysroot STREQUAL "")
+            message(FATAL_ERROR "Failed to determine toolchain sysroot")
+        endif()
+        string(STRIP "${compiler_sysroot}" compiler_sysroot)
+        get_filename_component(compiler_sysroot "${compiler_sysroot}/.." REALPATH)
+        list(APPEND compile_options "-fdebug-prefix-map=${compiler_sysroot}=/TOOLCHAIN")
+        string(APPEND gdbinit_file_lines "set substitute-path /TOOLCHAIN ${compiler_sysroot}\n")
+
+        # Write the final gdbinit file
+        set(gdbinit_path "${BUILD_DIR}/prefix_map_gdbinit")
+        file(WRITE "${gdbinit_path}" "${gdbinit_file_lines}")
+        idf_build_set_property(DEBUG_PREFIX_MAP_GDBINIT "${gdbinit_path}")
+    endif()
+
+    set(${compile_options_var} ${compile_options} PARENT_SCOPE)
+endfunction()

tools/test_build_system/conftest.py

@@ -95,7 +95,9 @@ def test_app_copy(func_work_dir: Path, request: FixtureRequest) -> typing.Genera
     # by default, use hello_world app and copy it to a temporary directory with
     # the name resembling that of the test
     copy_from = 'tools/test_build_system/build_test_app'
-    copy_to = request.node.name + '_app'
+    # sanitize test name in case pytest.mark.parametrize was used
+    test_name_sanitized = request.node.name.replace('[', '_').replace(']', '')
+    copy_to = test_name_sanitized + '_app'
 
     # allow overriding source and destination via pytest.mark.test_app_copy()
     mark = request.node.get_closest_marker('test_app_copy')

tools/test_build_system/test_build_system_helpers/__init__.py

@@ -1,15 +1,30 @@
-# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-from .build_constants import ALL_ARTIFACTS, APP_BINS, BOOTLOADER_BINS, JSON_METADATA, PARTITION_BIN
-from .editing import append_to_file, replace_in_file
-from .idf_utils import (EXT_IDF_PATH, EnvDict, IdfPyFunc, bin_file_contains, file_contains, find_python,
-                        get_idf_build_env, run_cmake, run_cmake_and_build, run_idf_py)
-from .snapshot import Snapshot, get_snapshot
+from .build_constants import ALL_ARTIFACTS
+from .build_constants import APP_BINS
+from .build_constants import BOOTLOADER_BINS
+from .build_constants import JSON_METADATA
+from .build_constants import PARTITION_BIN
+from .file_utils import append_to_file
+from .file_utils import bin_file_contains
+from .file_utils import bin_files_differ
+from .file_utils import file_contains
+from .file_utils import replace_in_file
+from .idf_utils import EnvDict
+from .idf_utils import EXT_IDF_PATH
+from .idf_utils import find_python
+from .idf_utils import get_idf_build_env
+from .idf_utils import IdfPyFunc
+from .idf_utils import run_cmake
+from .idf_utils import run_cmake_and_build
+from .idf_utils import run_idf_py
+from .snapshot import get_snapshot
+from .snapshot import Snapshot
 
 __all__ = [
     'append_to_file', 'replace_in_file',
     'get_idf_build_env', 'run_idf_py', 'EXT_IDF_PATH', 'EnvDict', 'IdfPyFunc',
     'Snapshot', 'get_snapshot', 'run_cmake', 'APP_BINS', 'BOOTLOADER_BINS',
     'PARTITION_BIN', 'JSON_METADATA', 'ALL_ARTIFACTS',
-    'run_cmake_and_build', 'find_python', 'file_contains', 'bin_file_contains'
+    'run_cmake_and_build', 'find_python', 'file_contains', 'bin_file_contains', 'bin_files_differ'
 ]