scripts: add Zephyr inline code generation with Python

CMakeLists.txt

@@ -336,7 +336,7 @@ add_subdirectory(misc)
 # property which is set implicitly for custom command outputs
 include(misc/generated/CMakeLists.txt)
 
-if(EXISTS soc/${ARCH}/CMakeLists.txt)
+if(EXISTS ${ZEPHYR_BASE}/soc/${ARCH}/CMakeLists.txt)
   add_subdirectory(${SOC_DIR}/${ARCH} soc/${ARCH})
 else()
   add_subdirectory(${SOC_DIR}/${ARCH}/${SOC_PATH} soc/${ARCH}/${SOC_PATH})
@@ -348,6 +348,13 @@ add_subdirectory(subsys)
 add_subdirectory(drivers)
 add_subdirectory(tests)
 
+# Add all generated zephyr sources in the same context as the zephyr library
+# is created. Assure all sub directories that might invoke code generation
+# are processed before.
+get_property(zephyr_generated_sources GLOBAL PROPERTY zephyr_generated_sources_property)
+set_source_files_properties(${zephyr_generated_sources} PROPERTIES GENERATED 1)
+target_sources(zephyr PRIVATE ${zephyr_generated_sources})
+
 set(syscall_macros_h ${ZEPHYR_BINARY_DIR}/include/generated/syscall_macros.h)
 
 add_custom_target(syscall_macros_h_target DEPENDS ${syscall_macros_h})

cmake/dts.cmake

@@ -10,6 +10,7 @@
 # See ~/zephyr/doc/dts
 set(GENERATED_DTS_BOARD_H    ${PROJECT_BINARY_DIR}/include/generated/generated_dts_board.h)
 set(GENERATED_DTS_BOARD_CONF ${PROJECT_BINARY_DIR}/include/generated/generated_dts_board.conf)
+set(GENERATED_EDTS ${PROJECT_BINARY_DIR}/edts.json)
 set_ifndef(DTS_SOURCE ${BOARD_ROOT}/boards/${ARCH}/${BOARD_FAMILY}/${BOARD}.dts)
 set_ifndef(DTS_COMMON_OVERLAYS ${ZEPHYR_BASE}/dts/common/common.dts)
 set_ifndef(DTS_APP_BINDINGS ${APPLICATION_SOURCE_DIR}/dts/bindings)
@@ -113,6 +114,7 @@ if(CONFIG_HAS_DTS)
     ${DTS_FIXUPS}
     --keyvalue ${GENERATED_DTS_BOARD_CONF}
     --include ${GENERATED_DTS_BOARD_H}
+    --edts ${GENERATED_EDTS}
     )
 
   # Run extract_dts_includes.py to create a .conf and a header file that can be

cmake/extensions.cmake

@@ -344,6 +344,17 @@ function(generate_inc_file_for_gen_target
   add_dependencies(${target} ${gen_target})
 endfunction()
 
+function(get_unique_generated_target_name
+    generated_file  # The generated file
+    generated_target_name # the unique name
+    )
+    string(RANDOM LENGTH 8 random_chars)
+    get_filename_component(basename ${generated_file} NAME)
+    string(REPLACE "." "_" basename ${basename})
+    string(REPLACE "@" "_" basename ${basename})
+    set(generated_target_name "gen_${basename}_${random_chars}" PARENT_SCOPE)
+endfunction()
+
 function(generate_inc_file_for_target
     target          # The cmake target that depends on the generated file
     source_file     # The source file to be converted to hex
@@ -353,22 +364,131 @@ function(generate_inc_file_for_target
   # Ensure 'generated_file' is generated before 'target' by creating a
   # 'custom_target' for it and setting up a dependency between the two
   # targets
+  get_unique_generated_target_name(${generated_file} generated_target_name)
+  add_custom_target(${generated_target_name} DEPENDS ${generated_file})
+  generate_inc_file_for_gen_target(${target} ${source_file} ${generated_file}
+                                   ${generated_target_name} ${ARGN})
+endfunction()
 
-  # But first create a unique name for the custom target
-  string(
-    RANDOM
-    LENGTH 8
-    random_chars
+function(target_sources_codegen
+    target          # The cmake target that depends on the generated file
     )
+  set(options)
+  set(oneValueArgs)
+  set(multiValueArgs CODEGEN_DEFINES DEPENDS)
+  cmake_parse_arguments(CODEGEN "${options}" "${oneValueArgs}"
+                        "${multiValueArgs}" ${ARGN})
+  # prepend -D to all defines
+  string(REGEX REPLACE "([^;]+)" "-D;\\1"
+         CODEGEN_CODEGEN_DEFINES "${CODEGEN_CODEGEN_DEFINES}")
+  # Get all the files that make up codegen for dependency
+  file(GLOB CODEGEN_SOURCES
+    ${ZEPHYR_BASE}/scripts/dts/edtsdatabase.py
+    ${ZEPHYR_BASE}/scripts/codegen/modules/*.py
+    ${ZEPHYR_BASE}/scripts/codegen/templates/drivers/*.py
+    ${ZEPHYR_BASE}/scripts/codegen/*.py
+    ${ZEPHYR_BASE}/scripts/gen_code.py)
+
+  message(STATUS "Will generate for target ${target}")
+  # Generated file must be generated to the current binary directory.
+  # Otherwise this would trigger CMake issue #14633:
+  # https://gitlab.kitware.com/cmake/cmake/issues/14633
+  foreach(arg ${CODEGEN_UNPARSED_ARGUMENTS})
+    if(IS_ABSOLUTE ${arg})
+      set(template_file ${arg})
+      get_filename_component(generated_file_name ${arg} NAME)
+      set(generated_file ${CMAKE_CURRENT_BINARY_DIR}/${generated_file_name})
+    else()
+      set(template_file ${CMAKE_CURRENT_SOURCE_DIR}/${arg})
+      set(generated_file ${CMAKE_CURRENT_BINARY_DIR}/${arg})
+    endif()
+    get_filename_component(template_dir ${template_file} DIRECTORY)
 
-  get_filename_component(basename ${generated_file} NAME)
-  string(REPLACE "." "_" basename ${basename})
-  string(REPLACE "@" "_" basename ${basename})
+    if(IS_DIRECTORY ${template_file})
+      message(FATAL_ERROR "target_sources_codegen() was called on a directory")
+    endif()
 
-  set(generated_target_name "gen_${basename}_${random_chars}")
+    # Generate file from template
+    message(STATUS " from '${template_file}'")
+    message(STATUS " to   '${generated_file}'")
+    add_custom_command(
+      COMMENT "CodeGen ${generated_file}"
+      OUTPUT ${generated_file}
+      MAIN_DEPENDENCY ${template_file}
+      DEPENDS
+      ${CODEGEN_DEPENDS}
+      ${CODEGEN_SOURCES}
+      COMMAND
+      ${PYTHON_EXECUTABLE}
+      ${ZEPHYR_BASE}/scripts/gen_code.py
+      ${CODEGEN_CODEGEN_DEFINES}
+      -D "PROJECT_NAME=${PROJECT_NAME}"
+      -D "PROJECT_SOURCE_DIR=${PROJECT_SOURCE_DIR}"
+      -D "PROJECT_BINARY_DIR=${PROJECT_BINARY_DIR}"
+      -D "CMAKE_SOURCE_DIR=${CMAKE_SOURCE_DIR}"
+      -D "CMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}"
+      -D "CMAKE_CURRENT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}"
+      -D "CMAKE_CURRENT_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}"
+      -D "CMAKE_CURRENT_LIST_DIR=${CMAKE_CURRENT_LIST_DIR}"
+      -D "CMAKE_FILES_DIRECTORY=${CMAKE_FILES_DIRECTORY}"
+      -D "CMAKE_PROJECT_NAME=${CMAKE_PROJECT_NAME}"
+      -D "CMAKE_SYSTEM=${CMAKE_SYSTEM}"
+      -D "CMAKE_SYSTEM_NAME=${CMAKE_SYSTEM_NAME}"
+      -D "CMAKE_SYSTEM_VERSION=${CMAKE_SYSTEM_VERSION}"
+      -D "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}"
+      -D "CMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
+      -D "CMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
+      -D "CMAKE_COMPILER_IS_GNUCC=${CMAKE_COMPILER_IS_GNUCC}"
+      -D "CMAKE_COMPILER_IS_GNUCXX=${CMAKE_COMPILER_IS_GNUCXX}"
+      -D "GENERATED_DTS_BOARD_H=${GENERATED_DTS_BOARD_H}"
+      -D "GENERATED_DTS_BOARD_CONF=${GENERATED_DTS_BOARD_CONF}"
+      -D "GENERATED_DTS_BOARD_EDTS=${GENERATED_DTS_BOARD_EDTS}"
+      --input "${template_file}"
+      --output "${generated_file}"
+      --log "${CMAKE_BINARY_DIR}/${CMAKE_FILES_DIRECTORY}/CodeGen.log"
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    )
+    if("${target}" STREQUAL "zephyr")
+      # zephyr is special
+      get_unique_generated_target_name(${generated_file} generated_target_name)
+      add_custom_target(${generated_target_name} DEPENDS ${generated_file})
+      add_dependencies(zephyr ${generated_target_name})
+      # Remember all the files that are generated for zephyr.
+      # target_sources(zephyr PRIVATE ${zephyr_generated_sources})
+      # is executed in the top level CMakeFile.txt context.
+      get_property(zephyr_generated_sources GLOBAL PROPERTY zephyr_generated_sources_property)
+      list(APPEND zephyr_generated_sources ${generated_file})
+      set_property(GLOBAL PROPERTY zephyr_generated_sources_property "${zephyr_generated_sources}")
+      # Add template directory to include path to allow includes with
+      # relative path in generated file to work
+      zephyr_include_directories(${template_dir})
+    else()
+      target_sources(${target} PRIVATE ${generated_file})
+      # Add template directory to include path to allow includes with
+      # relative path in generated file to work
+      target_include_directories(${target} PRIVATE ${template_dir})
+    endif()
+  endforeach()
+endfunction()
 
-  add_custom_target(${generated_target_name} DEPENDS ${generated_file})
-  generate_inc_file_for_gen_target(${target} ${source_file} ${generated_file} ${generated_target_name} ${ARGN})
+function(zephyr_sources_codegen)
+  target_sources_codegen(zephyr ${ARGN})
+endfunction()
+
+function(zephyr_sources_codegen_ifdef feature_toggle)
+  if(${${feature_toggle}})
+    zephyr_sources_codegen(${ARGN})
+  endif()
+endfunction()
+
+function(zephyr_library_sources_codegen)
+  target_sources_codegen(${ZEPHYR_CURRENT_LIBRARY} ${ARGN})
+endfunction()
+
+function(zephyr_library_sources_codegen_ifdef feature_toggle)
+  if(${${feature_toggle}})
+    zephyr_library_sources_codegen(${ARGN})
+  endif()
 endfunction()
 
 # 1.2 zephyr_library_*
@@ -424,6 +544,13 @@ macro(zephyr_library_named name)
   # This is a macro because we need add_library() to be executed
   # within the scope of the caller.
   set(ZEPHYR_CURRENT_LIBRARY ${name})
+
+  if("${name}" STREQUAL "zephyr")
+    # We have to mark all the generated files "GENERATED" in this context
+    get_property(zephyr_generated_files GLOBAL PROPERTY zephyr_generated_files_property)
+    set_source_files_properties(${zephyr_generated_files} PROPERTIES GENERATED 1)
+  endif()
+
   add_library(${name} STATIC "")
 
   zephyr_append_cmake_library(${name})

doc/contribute/contribute_guidelines.rst

@@ -326,8 +326,11 @@ On Linux systems, you can install uncrustify with
 For Windows installation instructions see the `sourceforge listing for
 uncrustify <https://sourceforge.net/projects/uncrustify>`_.
 
+Best coding practises
+*********************
+
 Coding Style
-************
+============
 
 Use these coding guidelines to ensure that your development complies with the
 project's style and naming conventions.
@@ -363,6 +366,25 @@ it to contain:
     set -e exec
     exec git diff --cached | ${ZEPHYR_BASE}/scripts/checkpatch.pl -
 
+Keep the code simple
+====================
+
+Keep the code as simple as possible.
+
+Code-generation preprocessing tools provide a convenient way to
+simplify some repetitive or parameterized coding tasks.  While Zephyr
+development allows use of such tools, we recommend keeping this
+use to a minimum and when it provides an appropriate and simple
+coding solution that follows these rules:
+
+* Use code generation - by preprocessor, :ref:`codegen`, or other - only for
+  problems that cannot be solved in the source language.
+* Limit code generation to declarative data. Avoid generating control logic
+  whenever possible.
+* Use the preprocessor for code generation as the primary tool.
+* Use :ref:`codegen` only if the preprocessor can not provide a simple solution.
+* Use CMake only if :ref:`codegen` can not be used.
+
 .. _Contribution workflow:
 
 Contribution Workflow

doc/subsystems/codegen/build.rst

@@ -0,0 +1,29 @@
+..
+    Copyright (c) 2018 Bobby Noelte
+    SPDX-License-Identifier: Apache-2.0
+
+.. _codegen_build:
+
+Code Generation in the Build Process
+####################################
+
+Code generation has to be invoked as part of the build process. Zephyr uses
+`CMake <https://cmake.org/>`_ as the tool to manage building the project.
+
+A file that contains inline code generation has to be added to the project
+by one of the following commands in a :file:`CMakeList.txt` file.
+
+.. function:: zephyr_sources_codegen(codegen_file.c [CODEGEN_DEFINES defines..] [DEPENDS target.. file..])
+
+.. function:: zephyr_sources_codegen_ifdef(ifguard codegen_file.c [CODEGEN_DEFINES defines..] [DEPENDS target.. file..])
+
+.. function:: zephyr_library_sources_codegen(codegen_file.c [CODEGEN_DEFINES defines..] [DEPENDS target.. file..])
+
+.. function:: zephyr_library_sources_codegen_ifdef(ifguard codegen_file.c [CODEGEN_DEFINES defines..] [DEPENDS target.. file..])
+
+The arguments given by the ``CODEGEN_DEFINES`` keyword have to be of the form
+``define_name=define_value``. The arguments become globals in the python
+snippets and can be accessed by ``define_name``.
+
+Dependencies given by the ``DEPENDS`` key word are added to the dependencies
+of the generated file.

doc/subsystems/codegen/codegen.rst

@@ -0,0 +1,90 @@
+..
+    Copyright (c) 2004-2015 Ned Batchelder
+    SPDX-License-Identifier: MIT
+    Copyright (c) 2018 Bobby Noelte
+    SPDX-License-Identifier: Apache-2.0
+
+.. _codegen_intro:
+
+Introduction
+############
+
+Python snippets that are inlined in a source file are used as code generators.
+The tool to scan the source file for the Python snippets and process them is
+Codegen. Codegen and part of this documentation is based on
+`Cog <https://nedbatchelder.com/code/cog/index.html>`_ from Ned Batchelder.
+
+The processing of source files is controlled by the CMake extension functions:
+zephyr_sources_codegen(..) or zephyr_library_sources_codegen(..). The generated
+source files are added to the Zephyr sources. During build the source files are
+processed by Codegen and the generated source files are written to the CMake
+binary directory.
+
+The inlined Python snippets can contain any Python code, they are regular
+Python scripts. All Python snippets in a source file and all Python snippets of
+included template files are treated as a python script with a common set of
+global Python variables. Global data created in one snippet can be used in
+another snippet that is processed later on. This feature could be used, for
+example, to customize included template files.
+
+An inlined Python snippet can always access the codegen module. The codegen
+module encapsulates and provides all the functions to retrieve information
+(options, device tree properties, CMake variables, config properties) and to
+output the generated code.
+
+Codegen transforms files in a very simple way: it finds chunks of Python code
+embedded in them, executes the Python code, and places its output combined with
+the original file into the generated file. The original file can contain
+whatever text you like around the Python code. It will usually be source code.
+
+For example, if you run this file through Codegen:
+
+::
+
+    /* This is my C file. */
+    ...
+    /**
+     * @code{.codegen}
+     * fnames = ['DoSomething', 'DoAnotherThing', 'DoLastThing']
+     * for fn in fnames:
+     *     codegen.outl("void %s();" % fn)
+     * @endcode{.codegen}
+     */
+    /** @code{.codeins}@endcode */
+    ...
+
+it will come out like this:
+
+::
+
+    /* This is my C file. */
+    ...
+    /**
+     * @code{.codegen}
+     * fnames = ['DoSomething', 'DoAnotherThing', 'DoLastThing']
+     * for fn in fnames:
+     *     codegen.outl("void %s();" % fn)
+     * @endcode{.codegen}
+     */
+    void DoSomething();
+    void DoAnotherThing();
+    void DoLastThing();
+    /** @code{.codeins}@endcode */
+    ...
+
+Lines with ``@code{.codegen}`` or ``@code{.codeins}@endcode`` are marker lines.
+The lines between ``@code{.codegen}`` and ``@endcode{.codegen}`` are the
+generator Python code. The lines between ``@endcode{.codegen}`` and
+``@code{.codeins}@endcode`` are the output from the generator.
+
+When Codegen runs, it discards the last generated Python output, executes the
+generator Python code, and writes its generated output into the file. All text
+lines outside of the special markers are passed through unchanged.
+
+The Codegen marker lines can contain any text in addition to the marker tokens.
+This makes it possible to hide the generator Python code from the source file.
+
+In the sample above, the entire chunk of Python code is a C comment, so the
+Python code can be left in place while the file is treated as C code.
+
+