Compile time code generation - add relevant code to all example builds (plus one sample codegen moved)

.github/workflows/build.yaml

@@ -210,6 +210,10 @@ jobs:
                         BUILD_TYPE=sanitizers scripts/build/gn_gen.sh --args="$GN_ARGS" --export-compile-commands
                         BUILD_TYPE=sanitizers scripts/tests/gn_tests.sh
                     done
+            - name: Ensure codegen is done for sanitize
+              timeout-minutes: 45
+              run: |
+                  ./scripts/run_in_build_env.sh "./scripts/run_codegen_targets.sh out/sanitizers"
             - name: Clang-tidy validation
               timeout-minutes: 45
               run: |
@@ -401,6 +405,10 @@ jobs:
                       scripts/run_in_build_env.sh "ninja -C ./out/$BUILD_TYPE"
                       BUILD_TYPE=$BUILD_TYPE scripts/tests/gn_tests.sh
                   done
+            - name: Ensure codegen is done for sanitize
+              timeout-minutes: 45
+              run: |
+                  ./scripts/run_in_build_env.sh "./scripts/run_codegen_targets.sh out/default"
             - name: Clang-tidy validation
               timeout-minutes: 45
               run: |

.github/workflows/lint.yml

@@ -97,6 +97,13 @@ jobs:
                      if [ "$idl_file" = './examples/thermostat/thermostat-common/thermostat.matter' ]; then continue; fi
                      if [ "$idl_file" = './examples/window-app/common/window-app.matter' ]; then continue; fi
 
+                     # Test files are intentionally small and not spec-compilant, just parse-compliant
+                     if [ "$idl_file" = "./scripts/idl/tests/inputs/cluster_struct_attribute.matter" ]; then continue; fi
+                     if [ "$idl_file" = "./scripts/idl/tests/inputs/global_struct_attribute.matter" ]; then continue; fi
+                     if [ "$idl_file" = "./scripts/idl/tests/inputs/optional_argument.matter" ]; then continue; fi
+                     if [ "$idl_file" = "./scripts/idl/tests/inputs/several_clusters.matter" ]; then continue; fi
+                     if [ "$idl_file" = "./scripts/idl/tests/inputs/simple_attribute.matter" ]; then continue; fi
+
                      ./scripts/run_in_build_env.sh "./scripts/idl_lint.py --log-level warn $idl_file" >/dev/null || exit 1
                  done
 

.restyled.yaml

@@ -74,6 +74,7 @@ exclude:
     - "examples/chef/zzz_generated/**/*"
     - "examples/platform/nxp/k32w/k32w0/scripts/demo_generated_certs/**/*"
     - "integrations/cloudbuild/*.yaml" # uglier long command line content
+    - "scripts/run_codegen_targets.sh" # shellharden breaks for loops over command outputs
 
 
 changed_paths:

build/chip/chip_codegen.cmake

@@ -0,0 +1,70 @@
+# Run chip code generation. 
+#
+# Example usage:
+#   chip_codegen("app"
+#      INPUT     "some_file.matter"
+#      GENERATOR "bridge"
+#      OUTPUTS
+#            "bridge/OnOff.h"
+#            "bridge/LevelControl.h"
+#            "bridge/Switch.h"
+#            # ... more outputs
+#     OUTPUT_PATH  DIR_NAME_VAR
+#     OUTPUT_FILES  FILE_NAMES_VAR
+#   )
+#
+# Arguments:
+#   INPUT     - the name of the ".matter" file to use for generation
+#   GENERATOR - generator to use for codegen.py
+#   OUTPUTS   - EXPECTED output names. MUST match actual outputs
+# 
+#   OUTPUT_PATH  - [OUT] output variable will contain the directory where the
+#                  files will be generated
+#   OUTPUT_FILES - [OUT] output variable will contain the path of generated files.
+#                  suitable to be added within a build target
+#
+function(chip_codegen TARGET_NAME)
+    cmake_parse_arguments(ARG 
+         "" 
+         "INPUT;GENERATOR;OUTPUT_PATH;OUTPUT_FILES" 
+         "OUTPUTS" 
+         ${ARGN}
+    )
+
+    set(GEN_FOLDER "${CMAKE_BINARY_DIR}/gen/${TARGET_NAME}/${ARG_GENERATOR}")
+
+    string(REPLACE ";" "\n" OUTPUT_AS_NEWLINES "${ARG_OUTPUTS}")
+
+    file(MAKE_DIRECTORY "${GEN_FOLDER}")
+    file(GENERATE
+        OUTPUT "${GEN_FOLDER}/expected.outputs"
+        CONTENT "${OUTPUT_AS_NEWLINES}"
+    )
+
+
+    set(OUT_NAMES)
+    foreach(NAME IN LISTS ARG_OUTPUTS)
+        list(APPEND OUT_NAMES "${GEN_FOLDER}/${NAME}")
+    endforeach()
+
+    # Python is expected to be in the path
+    #
+    # find_package(Python3 REQUIRED)
+    add_custom_command(
+        OUTPUT "${OUT_NAMES}"
+        COMMAND "${CHIP_ROOT}/scripts/codegen.py"
+        ARGS "--generator" "${ARG_GENERATOR}"
+             "--output-dir" "${GEN_FOLDER}"
+             "--expected-outputs" "${GEN_FOLDER}/expected.outputs"
+             "${ARG_INPUT}"
+        DEPENDS
+            "${ARG_INPUT}"
+        VERBATIM
+    )
+
+    add_custom_target(${TARGET_NAME} DEPENDS "${OUT_NAMES}")
+
+    # Forward outputs to the parent
+    set(${ARG_OUTPUT_FILES} "${OUT_NAMES}" PARENT_SCOPE)
+    set(${ARG_OUTPUT_PATH} "${GEN_FOLDER}" PARENT_SCOPE)
+endfunction()

build/chip/esp32/esp32_codegen.cmake

@@ -0,0 +1,45 @@
+#
+#    Copyright (c) 2022 Project CHIP Authors
+#    All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+
+
+macro(chip_app_component_codegen IDL_NAME)
+  include("${CHIP_ROOT}/build/chip/chip_codegen.cmake")
+
+  # The IDF build system performs a two-pass expansion to determine
+  # component expansion. The first pass runs in script-mode
+  # to determine idf_component_register REQUIRES and PRIV_REQUIRES.
+  #
+  # We can only set up code generation during the 2nd pass
+  #
+  # see https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-guides/build-system.html
+  if (NOT CMAKE_BUILD_EARLY_EXPANSION)
+
+    chip_codegen(app-codegen
+      INPUT     "${IDL_NAME}"
+      GENERATOR "cpp-app"
+      OUTPUTS
+            "app/PluginApplicationCallbacks.h"
+      OUTPUT_PATH   APP_GEN_DIR
+      OUTPUT_FILES  APP_GEN_FILES
+    )
+
+    target_include_directories(${COMPONENT_LIB} PUBLIC "${APP_GEN_DIR}")
+
+    add_dependencies(${COMPONENT_LIB} app-codegen)
+  endif()
+endmacro()

examples/all-clusters-app/esp32/main/CMakeLists.txt

@@ -14,7 +14,6 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
-
 # (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.)
 # The list of src and include dirs must be in sync with that in all-clusters-app/esp32/main/component.mk
 set(PRIV_INCLUDE_DIRS_LIST
@@ -137,6 +136,12 @@ idf_component_register(PRIV_INCLUDE_DIRS ${PRIV_INCLUDE_DIRS_LIST}
                        EXCLUDE_SRCS ${EXCLUDE_SRCS_LIST}
                        PRIV_REQUIRES ${PRIV_REQUIRES_LIST})
 
+get_filename_component(CHIP_ROOT ${CMAKE_SOURCE_DIR}/third_party/connectedhomeip REALPATH)
+
+include("${CHIP_ROOT}/build/chip/esp32/esp32_codegen.cmake")
+
+chip_app_component_codegen("${CHIP_ROOT}/examples/all-clusters-app/all-clusters-common/all-clusters-app.matter")
+
 set_property(TARGET ${COMPONENT_LIB} PROPERTY CXX_STANDARD 17)
 target_compile_options(${COMPONENT_LIB} PRIVATE "-DCHIP_HAVE_CONFIG_H")
 target_compile_options(${COMPONENT_LIB} PUBLIC
@@ -145,7 +150,6 @@ target_compile_options(${COMPONENT_LIB} PUBLIC
 
 if (CONFIG_ENABLE_PW_RPC)
 
-get_filename_component(CHIP_ROOT ${CMAKE_SOURCE_DIR}/third_party/connectedhomeip REALPATH)
 
 set(PIGWEED_ROOT "${CHIP_ROOT}/third_party/pigweed/repo")
 include(${PIGWEED_ROOT}/pw_build/pigweed.cmake)

examples/all-clusters-minimal-app/esp32/main/CMakeLists.txt

@@ -128,6 +128,12 @@ idf_component_register(PRIV_INCLUDE_DIRS ${PRIV_INCLUDE_DIRS_LIST}
                        SRC_DIRS ${SRC_DIRS_LIST}
                        PRIV_REQUIRES ${PRIV_REQUIRES_LIST})
 
+get_filename_component(CHIP_ROOT ${CMAKE_SOURCE_DIR}/third_party/connectedhomeip REALPATH)
+
+include("${CHIP_ROOT}/build/chip/esp32/esp32_codegen.cmake")
+
+chip_app_component_codegen("${CHIP_ROOT}/examples/all-clusters-minimal-app/all-clusters-common/all-clusters-minimal-app.matter")
+
 set_property(TARGET ${COMPONENT_LIB} PROPERTY CXX_STANDARD 17)
 target_compile_options(${COMPONENT_LIB} PRIVATE "-DCHIP_HAVE_CONFIG_H")
 target_compile_options(${COMPONENT_LIB} PUBLIC
@@ -136,8 +142,6 @@ target_compile_options(${COMPONENT_LIB} PUBLIC
 
 if (CONFIG_ENABLE_PW_RPC)
 
-get_filename_component(CHIP_ROOT ${CMAKE_SOURCE_DIR}/third_party/connectedhomeip REALPATH)
-
 set(PIGWEED_ROOT "${CHIP_ROOT}/third_party/pigweed/repo")
 include(${PIGWEED_ROOT}/pw_build/pigweed.cmake)
 include(${PIGWEED_ROOT}/pw_protobuf_compiler/proto.cmake)

examples/bridge-app/esp32/main/CMakeLists.txt

@@ -54,6 +54,12 @@ idf_component_register(PRIV_INCLUDE_DIRS
                       "${CMAKE_SOURCE_DIR}/third_party/connectedhomeip/examples/providers"
                       PRIV_REQUIRES chip QRCode bt)
 
+get_filename_component(CHIP_ROOT ${CMAKE_SOURCE_DIR}/third_party/connectedhomeip REALPATH)
+
+include("${CHIP_ROOT}/build/chip/esp32/esp32_codegen.cmake")
+
+chip_app_component_codegen("${CHIP_ROOT}/examples/bridge-app/bridge-common/bridge-app.matter")
+
 set_property(TARGET ${COMPONENT_LIB} PROPERTY CXX_STANDARD 14)
 target_compile_options(${COMPONENT_LIB} PRIVATE "-DCHIP_HAVE_CONFIG_H")
 target_compile_options(${COMPONENT_LIB} PUBLIC

examples/chef/esp32/main/CMakeLists.txt

@@ -16,7 +16,7 @@
 #
 # (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.)
 
-get_filename_component(CHIP_ROOT ${CMAKE_CURRENT_LIST_DIR}/../../../.. REALPATH)
+get_filename_component(CHIP_ROOT ../third_party/connectedhomeip REALPATH)
 get_filename_component(CHEF ${CMAKE_CURRENT_SOURCE_DIR}/../../ REALPATH)
 get_filename_component(GEN_DIR ${CHEF}/out/${SAMPLE_NAME}/zap-generated REALPATH)
 
@@ -107,6 +107,9 @@ idf_component_register(PRIV_INCLUDE_DIRS
                       PRIV_REQUIRES chip nvs_flash bt console esp32_mbedtls QRCode tft screen-framework spidriver
                       SRC_DIRS ${SRC_DIRS_LIST})
 
+include("${CHIP_ROOT}/build/chip/esp32/esp32_codegen.cmake")
+chip_app_component_codegen("${CHEF}/devices/${SAMPLE_NAME}.matter")
+
 set_property(TARGET ${COMPONENT_LIB} PROPERTY CXX_STANDARD 17)
 target_compile_options(${COMPONENT_LIB} PRIVATE "-DCHIP_HAVE_CONFIG_H")
 target_compile_options(${COMPONENT_LIB} PUBLIC
@@ -115,8 +118,6 @@ target_compile_options(${COMPONENT_LIB} PUBLIC
 
 if (CONFIG_ENABLE_PW_RPC)
 
-get_filename_component(CHIP_ROOT ../third_party/connectedhomeip REALPATH)
-
 set(PIGWEED_ROOT "${CHIP_ROOT}/third_party/pigweed/repo")
 include(${PIGWEED_ROOT}/pw_build/pigweed.cmake)
 include(${PIGWEED_ROOT}/pw_protobuf_compiler/proto.cmake)

examples/dynamic-bridge-app/bridge-common/BUILD.gn

@@ -22,6 +22,7 @@ chip_data_model("dynamic-bridge-common") {
 
   zap_pregenerated_dir =
       "${chip_root}/zzz_generated/dynamic-bridge-app/zap-generated"
+
   is_server = true
 
   # TODO: the definition of DYNAMIC_ENDPOINT_COUNT needs find a common home!
@@ -30,7 +31,7 @@ chip_data_model("dynamic-bridge-common") {
 
 if (chip_enable_pw_rpc) {
   import("//build_overrides/pigweed.gni")
-  import("$dir_pw_protobuf_compiler/proto.gni")
+  IMPORT("$dir_pw_protobuf_compiler/proto.gni")
 
   pw_proto_library("bridge_service") {
     sources = [ "protos/bridge_service.proto" ]

examples/light-switch-app/esp32/main/CMakeLists.txt

@@ -14,8 +14,8 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
-# (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.)
 
+# (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.)
 idf_component_register(PRIV_INCLUDE_DIRS
                       "${CMAKE_SOURCE_DIR}/third_party/connectedhomeip/zzz_generated/light-switch-app"
                       "${CMAKE_CURRENT_LIST_DIR}/include"
@@ -61,6 +61,11 @@ idf_component_register(PRIV_INCLUDE_DIRS
                       "${CMAKE_SOURCE_DIR}/third_party/connectedhomeip/src/app/clusters/groups-server"
                       "${CMAKE_SOURCE_DIR}/third_party/connectedhomeip/src/app/clusters/group-key-mgmt-server"
                       PRIV_REQUIRES chip QRCode bt app_update)
+get_filename_component(CHIP_ROOT ${CMAKE_SOURCE_DIR}/third_party/connectedhomeip REALPATH)
+
+include("${CHIP_ROOT}/build/chip/esp32/esp32_codegen.cmake")
+
+chip_app_component_codegen("${CHIP_ROOT}/examples/light-switch-app/light-switch-common/light-switch-app.matter")
 
 set_property(TARGET ${COMPONENT_LIB} PROPERTY CXX_STANDARD 17)
 target_compile_options(${COMPONENT_LIB} PRIVATE "-DCHIP_HAVE_CONFIG_H")

examples/lighting-app/esp32/main/CMakeLists.txt

@@ -17,12 +17,13 @@
 # (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.)
 
 set(PRIV_INCLUDE_DIRS_LIST
+                      "${APP_GEN_DIR}"
                       "${CMAKE_SOURCE_DIR}/third_party/connectedhomeip/zzz_generated/lighting-app"
                       "${CMAKE_SOURCE_DIR}/third_party/connectedhomeip/examples/providers"
                       "${CMAKE_SOURCE_DIR}/third_party/connectedhomeip/examples/lighting-app/lighting-common/include"
                       "${CMAKE_CURRENT_LIST_DIR}/include"
                       "${CMAKE_SOURCE_DIR}/third_party/connectedhomeip/examples/platform/esp32"
-) 
+)
 set(SRC_DIRS_LIST
                       "${CMAKE_CURRENT_LIST_DIR}"
                       "${CMAKE_SOURCE_DIR}/third_party/connectedhomeip/zzz_generated/app-common/app-common/zap-generated/attributes"
@@ -90,6 +91,11 @@ endif (CONFIG_ENABLE_PW_RPC)
 idf_component_register(PRIV_INCLUDE_DIRS ${PRIV_INCLUDE_DIRS_LIST}
                        SRC_DIRS ${SRC_DIRS_LIST}
                        PRIV_REQUIRES ${PRIV_REQUIRES_LIST})
+get_filename_component(CHIP_ROOT ${CMAKE_SOURCE_DIR}/third_party/connectedhomeip REALPATH)
+
+include("${CHIP_ROOT}/build/chip/esp32/esp32_codegen.cmake")
+
+chip_app_component_codegen("${CHIP_ROOT}/examples/lighting-app/lighting-common/lighting-app.matter")
 
 set_property(TARGET ${COMPONENT_LIB} PROPERTY CXX_STANDARD 17)
 target_compile_options(${COMPONENT_LIB} PRIVATE "-DCHIP_HAVE_CONFIG_H")

examples/lock-app/esp32/main/CMakeLists.txt

@@ -16,6 +16,8 @@
 #
 # (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.)
 
+get_filename_component(CHIP_ROOT ${CMAKE_SOURCE_DIR}/third_party/connectedhomeip REALPATH)
+
 if (CONFIG_ENABLE_PW_RPC)
 idf_component_register(INCLUDE_DIRS
                      "${CMAKE_CURRENT_LIST_DIR}"
@@ -67,8 +69,7 @@ idf_component_register(INCLUDE_DIRS
                      "${CMAKE_SOURCE_DIR}/third_party/connectedhomeip/src/app/clusters/identify-server"
                      "${CMAKE_SOURCE_DIR}/third_party/connectedhomeip/src/app/clusters/groups-server"
                      PRIV_REQUIRES bt chip QRCode)
-
-get_filename_component(CHIP_ROOT ${CMAKE_SOURCE_DIR}/third_party/connectedhomeip REALPATH)
+add_dependencies(${COMPONENT_LIB} app-codegen)
 
 set(PIGWEED_ROOT "${CHIP_ROOT}/third_party/pigweed/repo")
 include(${PIGWEED_ROOT}/pw_build/pigweed.cmake)
@@ -186,10 +187,18 @@ idf_component_register(PRIV_INCLUDE_DIRS
                       "${CMAKE_SOURCE_DIR}/third_party/connectedhomeip/src/app/clusters/groups-server"
                       PRIV_REQUIRES chip QRCode bt)
 
+add_dependencies(${COMPONENT_LIB} app-codegen)
+
 set_property(TARGET ${COMPONENT_LIB} PROPERTY CXX_STANDARD 17)
 target_compile_options(${COMPONENT_LIB} PRIVATE "-DCHIP_HAVE_CONFIG_H")
 target_compile_options(${COMPONENT_LIB} PUBLIC
            "-DCHIP_ADDRESS_RESOLVE_IMPL_INCLUDE_HEADER=<lib/address_resolve/AddressResolve_DefaultImpl.h>"
 )
 
 endif (CONFIG_ENABLE_PW_RPC)
+
+get_filename_component(CHIP_ROOT ${CMAKE_SOURCE_DIR}/third_party/connectedhomeip REALPATH)
+
+include("${CHIP_ROOT}/build/chip/esp32/esp32_codegen.cmake")
+
+chip_app_component_codegen("${CHIP_ROOT}/examples/lock-app/lock-common/lock-app.matter")

examples/ota-provider-app/esp32/main/CMakeLists.txt

@@ -16,7 +16,7 @@
 #
 #
 # (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.)
-idf_component_register(PRIV_INCLUDE_DIRS 
+idf_component_register(PRIV_INCLUDE_DIRS
                       "${CMAKE_CURRENT_LIST_DIR}/include"
                       "${CMAKE_SOURCE_DIR}/third_party/connectedhomeip/zzz_generated/ota-provider-app/"
                       "${CMAKE_SOURCE_DIR}/third_party/connectedhomeip/examples/ota-provider-app"
@@ -59,6 +59,10 @@ idf_component_register(PRIV_INCLUDE_DIRS
                       "${CMAKE_SOURCE_DIR}/third_party/connectedhomeip/examples/ota-provider-app/ota-provider-common/BdxOtaSender.cpp"
                       PRIV_REQUIRES chip QRCode bt console spiffs)
 
+get_filename_component(CHIP_ROOT ${CMAKE_SOURCE_DIR}/third_party/connectedhomeip REALPATH)
+include("${CHIP_ROOT}/build/chip/esp32/esp32_codegen.cmake")
+chip_app_component_codegen("${CHIP_ROOT}/examples/ota-provider-app/ota-provider-common/ota-provider-app.matter")
+
 spiffs_create_partition_image(img_storage ${CMAKE_SOURCE_DIR}/spiffs_image FLASH_IN_PROJECT)
 set_property(TARGET ${COMPONENT_LIB} PROPERTY CXX_STANDARD 14)
 target_compile_options(${COMPONENT_LIB} PRIVATE "-DCHIP_HAVE_CONFIG_H")