lestarch: changes necessary to restrict fpp from regenerating on any …

.github/actions/spelling/expect.txt

@@ -347,6 +347,7 @@ dictgen
 dicts
 dictvalue
 differend
+difflib
 diffs
 diles
 dinkel
@@ -864,6 +865,7 @@ lightyellow
 lindent
 lineno
 lineroo
+lineterm
 linting
 linux
 LINUXSERIALDRIVER

cmake/autocoder/ai-shared.cmake

@@ -32,11 +32,11 @@ function(setup_ai_autocode_variant VARIANT_ARGS OUTPUT_DIR EXTRA_COMMANDS AC_INP
             BYPRODUCTS ${GENERATED_FILES}
             COMMAND ${CMAKE_COMMAND} -E env
             PYTHONPATH=${PYTHON_AUTOCODER_DIR}/src:${PYTHON_AUTOCODER_DIR}/utils
-            BUILD_ROOT="${FPRIME_BUILD_LOCATIONS_SEP}:${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/F-Prime"
-            FPRIME_AC_CONSTANTS_FILE="${FPRIME_AC_CONSTANTS_FILE}"
+            BUILD_ROOT=${FPRIME_BUILD_LOCATIONS_SEP}:${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/F-Prime
+            FPRIME_AC_CONSTANTS_FILE=${FPRIME_AC_CONSTANTS_FILE}
             PYTHON_AUTOCODER_DIR=${PYTHON_AUTOCODER_DIR}
             ${PYTHON} ${FPRIME_FRAMEWORK_PATH}/Autocoders/Python/bin/codegen.py -p ${OUTPUT_DIR} --build_root ${VARIANT_ARGS} ${AC_INPUT_FILE}
             COMMAND ${EXTRA_COMMANDS}
             DEPENDS ${AC_INPUT_FILE} ${FILE_DEPENDENCIES} ${FPRIME_AC_CONSTANTS_FILE} ${MODULE_DEPENDENCIES} ${CODEGEN_TARGET}
     )
-endfunction(setup_ai_autocode_variant)
+endfunction(setup_ai_autocode_variant)

cmake/autocoder/autocoder.cmake

@@ -44,16 +44,8 @@ endfunction()
 function (__memoize SOURCES)
     string(MD5 SOURCES_HASH "${SOURCES}")
     set(MEMO_FILE "${CMAKE_CURRENT_BINARY_DIR}/${AUTOCODER_NAME}.${SOURCES_HASH}.dep")
-
-    # Run the expensive action only if the input has changed
-    on_any_changed("${SOURCES}" CHANGED)
-    if(COMMAND regenerate_memo)
-        regenerate_memo(FORCE_REGENERATE "${MEMO_FILE}")
-    endif()
-    if (CHANGED OR FORCE_REGENERATE OR NOT EXISTS "${MEMO_FILE}")
-        if (CMAKE_DEBUG_OUTPUT)
-            message(STATUS "[Autocode/${AUTOCODER_NAME}] Regenerating memo '${MEMO_FILE}' because: (changed: ${CHANGED}, forced: ${FORCE_REGENERATE})")
-        endif()
+    regenerate_memo(FORCE_REGENERATE "${MEMO_FILE}" "${SOURCES}")
+    if (FORCE_REGENERATE)
         get_generated_files("${SOURCES}")
         get_dependencies("${SOURCES}")
         resolve_dependencies(MODULE_DEPENDENCIES ${MODULE_DEPENDENCIES})
@@ -138,7 +130,7 @@ function(run_ac AUTOCODER_CMAKE SOURCES GENERATED_SOURCES INFO_ONLY)
     endif()
     # Configure depends on this source file if it causes a change to module dependencies
     if (MODULE_DEPENDENCIES)
-        set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${SOURCES})
+	    set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CONSUMED_SOURCES})
     endif()
 
     # Return variables

cmake/autocoder/default.cmake

@@ -14,13 +14,40 @@ function(is_supported AC_INPUT_FILE)
     message(FATAL_ERROR "${AUTOCODER_CMAKE} must define 'is_supported'")
 endfunction(is_supported)
 
+
+####
+# `default_regenerate_helper`:
+#
+# Default regenerate helper, so that other regenerates can be built on top of this.
+####
+function(default_regenerate_helper MISSING_VAR CHANGED_VAR MEMO_FILE SOURCES_INPUT)
+    # Check for missing memo file
+    set(MEMO_MISSING TRUE)
+    if (EXISTS "${MEMO_FILE}")
+        set(MEMO_MISSING FALSE)
+    endif()
+    # Changed sources
+    on_any_changed("${SOURCES_INPUT}" CHANGED)
+    set(${MISSING_VAR} ${MEMO_MISSING} PARENT_SCOPE)
+    set(${CHANGED_VAR} ${CHANGED} PARENT_SCOPE)
+endfunction(default_regenerate_helper)
+
+
 ###
 # `regenerate_memo`:
 #
 # Default implementation does not regenerate memo upon request of autocoder.
 ####
-function(regenerate_memo OUTPUT MEMO_FILE)
+function(regenerate_memo OUTPUT MEMO_FILE SOURCES_INPUT)
     set(${OUTPUT} FALSE PARENT_SCOPE)
+    default_regenerate_helper(MEMO_MISSING CHANGED "${MEMO_FILE}" "${SOURCES_INPUT}")
+    # Regenerating on any of the above
+    if (MEMO_MISSING OR CHANGED)
+        if (CMAKE_DEBUG_OUTPUT)
+            message(STATUS "[Autocode/${AUTOCODER_NAME}] Regenerating memo '${MEMO_FILE}' because: (memo missing: ${MEMO_MISSING}, sources changed: ${CHANGED})")
+        endif()
+	set(${OUTPUT} TRUE PARENT_SCOPE)
+    endif()
 endfunction(regenerate_memo)
 
 ####

cmake/autocoder/fpp-locs-differ/fpp-locs-differ.py

@@ -0,0 +1,58 @@
+import os
+import sys
+import argparse
+import difflib
+from fprime_ac.utils.buildroot import (
+    build_root_relative_path,
+    set_build_roots,
+)
+
+
+def main():
+    """Process arguments: file old, file new, module deps to see if any mod deps has changed"""
+    set_build_roots(os.environ["BUILD_ROOT"])
+    parser = argparse.ArgumentParser(
+        description="Has a module dependency changed within the locs file."
+    )
+    parser.add_argument("new_locs", help="New locs file")
+    parser.add_argument("prev_locs", help="Original locs file")
+    parser.add_argument("file_deps", nargs="+", help="Module deps")
+
+    args_ns = parser.parse_args()
+
+    if not os.path.exists(args_ns.new_locs):
+        print(f"[ERROR] Failed to open {args_ns.new_locs}")
+        sys.exit(1)
+    # Previous files not generated
+    if not os.path.exists(args_ns.prev_locs):
+        print(f"No pervious locations")
+        sys.exit(1)
+
+    with open(args_ns.prev_locs, "r") as prev_locs_fh:
+        prev_lines = prev_locs_fh.readlines()
+    with open(args_ns.new_locs, "r") as new_locs_fh:
+        new_lines = new_locs_fh.readlines()
+
+    diff_output = list(difflib.unified_diff(prev_lines, new_lines, n=0, lineterm="\n"))
+    subtracted_lines = [
+        diff
+        for diff in diff_output
+        if diff.startswith("-") and not diff.startswith("---")
+    ]
+    relative_mod_paths = [line.split()[-1].strip('"') for line in subtracted_lines]
+    subtractions = [
+        os.path.abspath(os.path.join(os.path.dirname(args_ns.new_locs), rel_path))
+        for rel_path in relative_mod_paths
+    ]
+    changed = []
+    for subtraction in subtractions:
+        if subtraction in args_ns.file_deps:
+            changed.append(os.path.basename(subtraction))
+    if changed:
+        print(f"{', '.join(changed)}")
+        sys.exit(2)
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

cmake/autocoder/fpp-wrapper/fpp-run-or-remove

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Grab the removal file from the first argument
+REMOVE_FILE="$1"
+
+# Nab the program as the first argument, and shift it away
+PROGRAM="$2"
+shift
+shift
+
+# Run the wrapped program with the properly wrapped remaining arguments
+"${PROGRAM}" "$@"
+RET="$?"
+if (( $RET != 0 ))
+then
+    rm -f "${REMOVE_FILE}"
+fi
+exit $RET

cmake/autocoder/fpp.cmake

@@ -1,4 +1,8 @@
 include(utilities)
+include(autocoder/default)
+
+
+set(FPP_RUN_OR_REMOVE "${CMAKE_CURRENT_LIST_DIR}/fpp-wrapper/fpp-run-or-remove")
 
 set(HANDLES_INDIVIDUAL_SOURCES FALSE)
 
@@ -10,11 +14,44 @@ function(is_supported AC_INPUT_FILE)
     endif()
 endfunction(is_supported)
 
-#TODO: add a correction function w.r.t. removing items from topologies
+function(regenerate_memo OUTPUT MEMO_FILE SOURCES_INPUT)
+    # Initialize variables
+    set(${OUTPUT} FALSE PARENT_SCOPE)
+    set(LOCS_MOVED FALSE)
+    set(OUTPUT_TEXT "n/a")
+    set(ALL_FPP_INPUTS)
+    set(PREV_LOCS_FILE "${MEMO_FILE}.locs.prev")
+    default_regenerate_helper(MEMO_MISSING CHANGED "${MEMO_FILE}" "${SOURCES_INPUT}")
 
-function(regenerate_memo OUTPUT MEMO_FILE)
-    on_any_changed("${FPP_LOCS_FILE}" CHANGED) #TODO: fix this function
-    set(${OUTPUT} "${CHANGED}" PARENT_SCOPE)
+    # Read the memo file for dependencies, should it exist
+    if (NOT MEMO_MISSING)
+        file(READ "${MEMO_FILE}" CONTENTS)
+        read_from_lines("${CONTENTS}" GENERATED_FILES MODULE_DEPENDENCIES FILE_DEPENDENCIES ALL_FPP_INPUTS)
+    endif()
+    set_property(GLOBAL PROPERTY "FPP_LOCATIONS_CACHE_${MODULE_NAME}" "${PREV_LOCS_FILE}")
+    set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${PREV_LOCS_FILE}")
+    # Look at changed inter-module dependencies
+    if (ALL_FPP_INPUTS)
+        # Subset module deps by what changed
+        string(REPLACE ";" ":" FPRIME_BUILD_LOCATIONS_SEP "${FPRIME_BUILD_LOCATIONS}")
+        find_program(PYTHON NAMES python3 python REQUIRED)
+        execute_process(COMMAND ${CMAKE_COMMAND} -E env 
+            PYTHONPATH=${PYTHON_AUTOCODER_DIR}/src:${PYTHON_AUTOCODER_DIR}/utils
+            BUILD_ROOT=${FPRIME_BUILD_LOCATIONS_SEP}:${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/F-Prime
+            ${PYTHON} "${FPRIME_FRAMEWORK_PATH}/cmake/autocoder/fpp-locs-differ/fpp-locs-differ.py" ${FPP_LOCS_FILE} ${PREV_LOCS_FILE}
+            ${ALL_FPP_INPUTS} OUTPUT_VARIABLE OUTPUT_TEXT OUTPUT_STRIP_TRAILING_WHITESPACE RESULT_VARIABLE RESULT_OUT)
+        if (NOT RESULT_OUT EQUAL "0")
+            set(LOCS_MOVED TRUE)
+        endif()
+    endif()
+    # Regenerating on any of the above
+    if (MEMO_MISSING OR CHANGED OR LOCS_MOVED)
+        execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different ${FPP_LOCS_FILE} ${PREV_LOCS_FILE}) 
+        if (CMAKE_DEBUG_OUTPUT)
+            message(STATUS "[Autocode/${AUTOCODER_NAME}] Regenerating memo '${MEMO_FILE}' because: (memo missing: ${MEMO_MISSING}, sources changed: ${CHANGED}, dependencies moved: ${LOCS_MOVED} (${OUTPUT_TEXT}))")
+        endif()
+	    set(${OUTPUT} TRUE PARENT_SCOPE)
+    endif()
 endfunction(regenerate_memo)
 
 function(get_generated_files AC_INPUT_FILES)
@@ -26,7 +63,8 @@ function(get_generated_files AC_INPUT_FILES)
     set(INCLUDED_FILE "${CMAKE_CURRENT_BINARY_DIR}/included.txt")
     set(MISSING_FILE "${CMAKE_CURRENT_BINARY_DIR}/missing.txt")
     set(GENERATED_FILE "${CMAKE_CURRENT_BINARY_DIR}/generated.txt")
-    set(LAST_DEP_COMMAND "${FPP_DEPEND} ${FPP_LOCS_FILE} ${AC_INPUT_FILES} -d ${DIRECT_FILE} -i ${INCLUDED_FILE} -m ${MISSING_FILE} -g ${GENERATED_FILE}" CACHE INTERNAL "Last command to annotate memo file" FORCE)
+    set(LAST_DEP_COMMAND "${FPP_DEPEND} ${FPP_LOCS_FILE} ${AC_INPUT_FILES} -d ${DIRECT_FILE} -i ${INCLUDED_FILE} -m ${MISSING_FILE} -g ${GENERATED_FILE}"
+        CACHE INTERNAL "Last command to annotate memo file" FORCE)
     execute_process(COMMAND ${FPP_DEPEND} ${FPP_LOCS_FILE} ${AC_INPUT_FILES}
         -d "${DIRECT_FILE}"
         -i "${INCLUDED_FILE}"
@@ -118,10 +156,11 @@ function(setup_autocode AC_INPUT_FILES GENERATED_FILES MODULE_DEPENDENCIES FILE_
     endforeach()
 
     # Add in steps for Ai.xml generation
+    get_property(REMOVAL_FILE GLOBAL PROPERTY "FPP_LOCATIONS_CACHE_${MODULE_NAME}")
     if (GENERATED_AI)
         add_custom_command(
                 OUTPUT  ${GENERATED_AI}
-                COMMAND ${FPP_TO_XML} ${AC_INPUT_FILES} "-d" "${CMAKE_CURRENT_BINARY_DIR}" ${INCLUDES}
+                COMMAND ${FPP_RUN_OR_REMOVE} ${REMOVAL_FILE} ${FPP_TO_XML} ${AC_INPUT_FILES} "-d" "${CMAKE_CURRENT_BINARY_DIR}" ${INCLUDES}
                     "-p" "${FPRIME_BUILD_LOCATIONS_SEP_FPP}"
                 DEPENDS ${AC_INPUT_FILE} ${FILE_DEPENDENCIES} ${MODULE_DEPENDENCIES}
         )
@@ -130,7 +169,7 @@ function(setup_autocode AC_INPUT_FILES GENERATED_FILES MODULE_DEPENDENCIES FILE_
     if (GENERATED_CPP)
         add_custom_command(
                 OUTPUT  ${GENERATED_CPP}
-                COMMAND ${FPP_TO_CPP} ${AC_INPUT_FILES} "-d" "${CMAKE_CURRENT_BINARY_DIR}" ${INCLUDES}
+                COMMAND ${FPP_RUN_OR_REMOVE} ${REMOVAL_FILE} ${FPP_TO_CPP} ${AC_INPUT_FILES} "-d" "${CMAKE_CURRENT_BINARY_DIR}" ${INCLUDES}
                 "-p" "${FPRIME_BUILD_LOCATIONS_SEP_FPP},${CMAKE_BINARY_DIR}"
                 DEPENDS ${AC_INPUT_FILE} ${FILE_DEPENDENCIES} ${MODULE_DEPENDENCIES}
         )

cmake/fpp-download/CMakeLists.fpp.txt.in

@@ -5,11 +5,11 @@ project(fpp-download NONE)
 include(ExternalProject)
 ExternalProject_Add(fpp-tools
   GIT_REPOSITORY    https://github.com/fprime-community/fpp
-  GIT_TAG           6b1b39634c40f76641287c5c2221b937bc97e058
+  GIT_TAG           ${FPP_VERSION}
   SOURCE_DIR        "${CMAKE_BINARY_DIR}/fpp-tools/src"
   BINARY_DIR        "${CMAKE_BINARY_DIR}/fpp-tools/bin"
   CONFIGURE_COMMAND ""
   BUILD_COMMAND     ""
-  INSTALL_COMMAND   "${CMAKE_BINARY_DIR}/fpp-tools/src/compiler/install"
+  INSTALL_COMMAND   "${CMAKE_BINARY_DIR}/fpp-tools/src/compiler/install" "${FPP_TOOLS_PATH}"
   TEST_COMMAND      ""
 )

cmake/fpp-download/CMakeLists.sbt.txt.in

@@ -6,7 +6,7 @@ include(ExternalProject)
 ExternalProject_Add(sbt
   URL https://github.com/sbt/sbt/releases/download/v1.5.5/sbt-1.5.5.zip
   DOWNLOAD_DIR      "${CMAKE_BINARY_DIR}/sbt-download"
-  SOURCE_DIR        "${CMAKE_BINARY_DIR}/sbt"
+  SOURCE_DIR        "${FPP_TOOLS_PATH}/sbt"
   CONFIGURE_COMMAND ""
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""