Skip to content

Commit

Permalink
lestarch: restrict fpp from regenerating on any locs change, fpp tool…
Browse files Browse the repository at this point in the history
…s verion checking (#1087)

* lestarch: changes necessary to restrict fpp from regenerating on any loc change

* lestarch: fixing prev locs hole on crash

* lestarch: formatting Python

* lestarch: appeasing CI

* lestarch: fpp tools installation checks version
  • Loading branch information
LeStarch authored Nov 4, 2021
1 parent 42a0fec commit 661234d
Show file tree
Hide file tree
Showing 12 changed files with 267 additions and 71 deletions.
2 changes: 2 additions & 0 deletions .github/actions/spelling/expect.txt
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,7 @@ dictgen
dicts
dictvalue
differend
difflib
diffs
diles
dinkel
Expand Down Expand Up @@ -864,6 +865,7 @@ lightyellow
lindent
lineno
lineroo
lineterm
linting
linux
LINUXSERIALDRIVER
Expand Down
6 changes: 3 additions & 3 deletions cmake/autocoder/ai-shared.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,11 @@ function(setup_ai_autocode_variant VARIANT_ARGS OUTPUT_DIR EXTRA_COMMANDS AC_INP
BYPRODUCTS ${GENERATED_FILES}
COMMAND ${CMAKE_COMMAND} -E env
PYTHONPATH=${PYTHON_AUTOCODER_DIR}/src:${PYTHON_AUTOCODER_DIR}/utils
BUILD_ROOT="${FPRIME_BUILD_LOCATIONS_SEP}:${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/F-Prime"
FPRIME_AC_CONSTANTS_FILE="${FPRIME_AC_CONSTANTS_FILE}"
BUILD_ROOT=${FPRIME_BUILD_LOCATIONS_SEP}:${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/F-Prime
FPRIME_AC_CONSTANTS_FILE=${FPRIME_AC_CONSTANTS_FILE}
PYTHON_AUTOCODER_DIR=${PYTHON_AUTOCODER_DIR}
${PYTHON} ${FPRIME_FRAMEWORK_PATH}/Autocoders/Python/bin/codegen.py -p ${OUTPUT_DIR} --build_root ${VARIANT_ARGS} ${AC_INPUT_FILE}
COMMAND ${EXTRA_COMMANDS}
DEPENDS ${AC_INPUT_FILE} ${FILE_DEPENDENCIES} ${FPRIME_AC_CONSTANTS_FILE} ${MODULE_DEPENDENCIES} ${CODEGEN_TARGET}
)
endfunction(setup_ai_autocode_variant)
endfunction(setup_ai_autocode_variant)
14 changes: 3 additions & 11 deletions cmake/autocoder/autocoder.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -44,16 +44,8 @@ endfunction()
function (__memoize SOURCES)
string(MD5 SOURCES_HASH "${SOURCES}")
set(MEMO_FILE "${CMAKE_CURRENT_BINARY_DIR}/${AUTOCODER_NAME}.${SOURCES_HASH}.dep")

# Run the expensive action only if the input has changed
on_any_changed("${SOURCES}" CHANGED)
if(COMMAND regenerate_memo)
regenerate_memo(FORCE_REGENERATE "${MEMO_FILE}")
endif()
if (CHANGED OR FORCE_REGENERATE OR NOT EXISTS "${MEMO_FILE}")
if (CMAKE_DEBUG_OUTPUT)
message(STATUS "[Autocode/${AUTOCODER_NAME}] Regenerating memo '${MEMO_FILE}' because: (changed: ${CHANGED}, forced: ${FORCE_REGENERATE})")
endif()
regenerate_memo(FORCE_REGENERATE "${MEMO_FILE}" "${SOURCES}")
if (FORCE_REGENERATE)
get_generated_files("${SOURCES}")
get_dependencies("${SOURCES}")
resolve_dependencies(MODULE_DEPENDENCIES ${MODULE_DEPENDENCIES})
Expand Down Expand Up @@ -138,7 +130,7 @@ function(run_ac AUTOCODER_CMAKE SOURCES GENERATED_SOURCES INFO_ONLY)
endif()
# Configure depends on this source file if it causes a change to module dependencies
if (MODULE_DEPENDENCIES)
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${SOURCES})
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CONSUMED_SOURCES})
endif()

# Return variables
Expand Down
29 changes: 28 additions & 1 deletion cmake/autocoder/default.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,40 @@ function(is_supported AC_INPUT_FILE)
message(FATAL_ERROR "${AUTOCODER_CMAKE} must define 'is_supported'")
endfunction(is_supported)


####
# `default_regenerate_helper`:
#
# Default regenerate helper, so that other regenerates can be built on top of this.
####
function(default_regenerate_helper MISSING_VAR CHANGED_VAR MEMO_FILE SOURCES_INPUT)
# Check for missing memo file
set(MEMO_MISSING TRUE)
if (EXISTS "${MEMO_FILE}")
set(MEMO_MISSING FALSE)
endif()
# Changed sources
on_any_changed("${SOURCES_INPUT}" CHANGED)
set(${MISSING_VAR} ${MEMO_MISSING} PARENT_SCOPE)
set(${CHANGED_VAR} ${CHANGED} PARENT_SCOPE)
endfunction(default_regenerate_helper)


###
# `regenerate_memo`:
#
# Default implementation does not regenerate memo upon request of autocoder.
####
function(regenerate_memo OUTPUT MEMO_FILE)
function(regenerate_memo OUTPUT MEMO_FILE SOURCES_INPUT)
set(${OUTPUT} FALSE PARENT_SCOPE)
default_regenerate_helper(MEMO_MISSING CHANGED "${MEMO_FILE}" "${SOURCES_INPUT}")
# Regenerating on any of the above
if (MEMO_MISSING OR CHANGED)
if (CMAKE_DEBUG_OUTPUT)
message(STATUS "[Autocode/${AUTOCODER_NAME}] Regenerating memo '${MEMO_FILE}' because: (memo missing: ${MEMO_MISSING}, sources changed: ${CHANGED})")
endif()
set(${OUTPUT} TRUE PARENT_SCOPE)
endif()
endfunction(regenerate_memo)

####
Expand Down
58 changes: 58 additions & 0 deletions cmake/autocoder/fpp-locs-differ/fpp-locs-differ.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import os
import sys
import argparse
import difflib
from fprime_ac.utils.buildroot import (
build_root_relative_path,
set_build_roots,
)


def main():
"""Process arguments: file old, file new, module deps to see if any mod deps has changed"""
set_build_roots(os.environ["BUILD_ROOT"])
parser = argparse.ArgumentParser(
description="Has a module dependency changed within the locs file."
)
parser.add_argument("new_locs", help="New locs file")
parser.add_argument("prev_locs", help="Original locs file")
parser.add_argument("file_deps", nargs="+", help="Module deps")

args_ns = parser.parse_args()

if not os.path.exists(args_ns.new_locs):
print(f"[ERROR] Failed to open {args_ns.new_locs}")
sys.exit(1)
# Previous files not generated
if not os.path.exists(args_ns.prev_locs):
print(f"No pervious locations")
sys.exit(1)

with open(args_ns.prev_locs, "r") as prev_locs_fh:
prev_lines = prev_locs_fh.readlines()
with open(args_ns.new_locs, "r") as new_locs_fh:
new_lines = new_locs_fh.readlines()

diff_output = list(difflib.unified_diff(prev_lines, new_lines, n=0, lineterm="\n"))
subtracted_lines = [
diff
for diff in diff_output
if diff.startswith("-") and not diff.startswith("---")
]
relative_mod_paths = [line.split()[-1].strip('"') for line in subtracted_lines]
subtractions = [
os.path.abspath(os.path.join(os.path.dirname(args_ns.new_locs), rel_path))
for rel_path in relative_mod_paths
]
changed = []
for subtraction in subtractions:
if subtraction in args_ns.file_deps:
changed.append(os.path.basename(subtraction))
if changed:
print(f"{', '.join(changed)}")
sys.exit(2)
sys.exit(0)


if __name__ == "__main__":
main()
18 changes: 18 additions & 0 deletions cmake/autocoder/fpp-wrapper/fpp-run-or-remove
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
#!/bin/bash

# Grab the removal file from the first argument
REMOVE_FILE="$1"

# Nab the program as the first argument, and shift it away
PROGRAM="$2"
shift
shift

# Run the wrapped program with the properly wrapped remaining arguments
"${PROGRAM}" "$@"
RET="$?"
if (( $RET != 0 ))
then
rm -f "${REMOVE_FILE}"
fi
exit $RET
53 changes: 46 additions & 7 deletions cmake/autocoder/fpp.cmake
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
include(utilities)
include(autocoder/default)


set(FPP_RUN_OR_REMOVE "${CMAKE_CURRENT_LIST_DIR}/fpp-wrapper/fpp-run-or-remove")

set(HANDLES_INDIVIDUAL_SOURCES FALSE)

Expand All @@ -10,11 +14,44 @@ function(is_supported AC_INPUT_FILE)
endif()
endfunction(is_supported)

#TODO: add a correction function w.r.t. removing items from topologies
function(regenerate_memo OUTPUT MEMO_FILE SOURCES_INPUT)
# Initialize variables
set(${OUTPUT} FALSE PARENT_SCOPE)
set(LOCS_MOVED FALSE)
set(OUTPUT_TEXT "n/a")
set(ALL_FPP_INPUTS)
set(PREV_LOCS_FILE "${MEMO_FILE}.locs.prev")
default_regenerate_helper(MEMO_MISSING CHANGED "${MEMO_FILE}" "${SOURCES_INPUT}")

function(regenerate_memo OUTPUT MEMO_FILE)
on_any_changed("${FPP_LOCS_FILE}" CHANGED) #TODO: fix this function
set(${OUTPUT} "${CHANGED}" PARENT_SCOPE)
# Read the memo file for dependencies, should it exist
if (NOT MEMO_MISSING)
file(READ "${MEMO_FILE}" CONTENTS)
read_from_lines("${CONTENTS}" GENERATED_FILES MODULE_DEPENDENCIES FILE_DEPENDENCIES ALL_FPP_INPUTS)
endif()
set_property(GLOBAL PROPERTY "FPP_LOCATIONS_CACHE_${MODULE_NAME}" "${PREV_LOCS_FILE}")
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${PREV_LOCS_FILE}")
# Look at changed inter-module dependencies
if (ALL_FPP_INPUTS)
# Subset module deps by what changed
string(REPLACE ";" ":" FPRIME_BUILD_LOCATIONS_SEP "${FPRIME_BUILD_LOCATIONS}")
find_program(PYTHON NAMES python3 python REQUIRED)
execute_process(COMMAND ${CMAKE_COMMAND} -E env
PYTHONPATH=${PYTHON_AUTOCODER_DIR}/src:${PYTHON_AUTOCODER_DIR}/utils
BUILD_ROOT=${FPRIME_BUILD_LOCATIONS_SEP}:${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/F-Prime
${PYTHON} "${FPRIME_FRAMEWORK_PATH}/cmake/autocoder/fpp-locs-differ/fpp-locs-differ.py" ${FPP_LOCS_FILE} ${PREV_LOCS_FILE}
${ALL_FPP_INPUTS} OUTPUT_VARIABLE OUTPUT_TEXT OUTPUT_STRIP_TRAILING_WHITESPACE RESULT_VARIABLE RESULT_OUT)
if (NOT RESULT_OUT EQUAL "0")
set(LOCS_MOVED TRUE)
endif()
endif()
# Regenerating on any of the above
if (MEMO_MISSING OR CHANGED OR LOCS_MOVED)
execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different ${FPP_LOCS_FILE} ${PREV_LOCS_FILE})
if (CMAKE_DEBUG_OUTPUT)
message(STATUS "[Autocode/${AUTOCODER_NAME}] Regenerating memo '${MEMO_FILE}' because: (memo missing: ${MEMO_MISSING}, sources changed: ${CHANGED}, dependencies moved: ${LOCS_MOVED} (${OUTPUT_TEXT}))")
endif()
set(${OUTPUT} TRUE PARENT_SCOPE)
endif()
endfunction(regenerate_memo)

function(get_generated_files AC_INPUT_FILES)
Expand All @@ -26,7 +63,8 @@ function(get_generated_files AC_INPUT_FILES)
set(INCLUDED_FILE "${CMAKE_CURRENT_BINARY_DIR}/included.txt")
set(MISSING_FILE "${CMAKE_CURRENT_BINARY_DIR}/missing.txt")
set(GENERATED_FILE "${CMAKE_CURRENT_BINARY_DIR}/generated.txt")
set(LAST_DEP_COMMAND "${FPP_DEPEND} ${FPP_LOCS_FILE} ${AC_INPUT_FILES} -d ${DIRECT_FILE} -i ${INCLUDED_FILE} -m ${MISSING_FILE} -g ${GENERATED_FILE}" CACHE INTERNAL "Last command to annotate memo file" FORCE)
set(LAST_DEP_COMMAND "${FPP_DEPEND} ${FPP_LOCS_FILE} ${AC_INPUT_FILES} -d ${DIRECT_FILE} -i ${INCLUDED_FILE} -m ${MISSING_FILE} -g ${GENERATED_FILE}"
CACHE INTERNAL "Last command to annotate memo file" FORCE)
execute_process(COMMAND ${FPP_DEPEND} ${FPP_LOCS_FILE} ${AC_INPUT_FILES}
-d "${DIRECT_FILE}"
-i "${INCLUDED_FILE}"
Expand Down Expand Up @@ -118,10 +156,11 @@ function(setup_autocode AC_INPUT_FILES GENERATED_FILES MODULE_DEPENDENCIES FILE_
endforeach()

# Add in steps for Ai.xml generation
get_property(REMOVAL_FILE GLOBAL PROPERTY "FPP_LOCATIONS_CACHE_${MODULE_NAME}")
if (GENERATED_AI)
add_custom_command(
OUTPUT ${GENERATED_AI}
COMMAND ${FPP_TO_XML} ${AC_INPUT_FILES} "-d" "${CMAKE_CURRENT_BINARY_DIR}" ${INCLUDES}
COMMAND ${FPP_RUN_OR_REMOVE} ${REMOVAL_FILE} ${FPP_TO_XML} ${AC_INPUT_FILES} "-d" "${CMAKE_CURRENT_BINARY_DIR}" ${INCLUDES}
"-p" "${FPRIME_BUILD_LOCATIONS_SEP_FPP}"
DEPENDS ${AC_INPUT_FILE} ${FILE_DEPENDENCIES} ${MODULE_DEPENDENCIES}
)
Expand All @@ -130,7 +169,7 @@ function(setup_autocode AC_INPUT_FILES GENERATED_FILES MODULE_DEPENDENCIES FILE_
if (GENERATED_CPP)
add_custom_command(
OUTPUT ${GENERATED_CPP}
COMMAND ${FPP_TO_CPP} ${AC_INPUT_FILES} "-d" "${CMAKE_CURRENT_BINARY_DIR}" ${INCLUDES}
COMMAND ${FPP_RUN_OR_REMOVE} ${REMOVAL_FILE} ${FPP_TO_CPP} ${AC_INPUT_FILES} "-d" "${CMAKE_CURRENT_BINARY_DIR}" ${INCLUDES}
"-p" "${FPRIME_BUILD_LOCATIONS_SEP_FPP},${CMAKE_BINARY_DIR}"
DEPENDS ${AC_INPUT_FILE} ${FILE_DEPENDENCIES} ${MODULE_DEPENDENCIES}
)
Expand Down
4 changes: 2 additions & 2 deletions cmake/fpp-download/CMakeLists.fpp.txt.in
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@ project(fpp-download NONE)
include(ExternalProject)
ExternalProject_Add(fpp-tools
GIT_REPOSITORY https://github.com/fprime-community/fpp
GIT_TAG 6b1b39634c40f76641287c5c2221b937bc97e058
GIT_TAG ${FPP_VERSION}
SOURCE_DIR "${CMAKE_BINARY_DIR}/fpp-tools/src"
BINARY_DIR "${CMAKE_BINARY_DIR}/fpp-tools/bin"
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND "${CMAKE_BINARY_DIR}/fpp-tools/src/compiler/install"
INSTALL_COMMAND "${CMAKE_BINARY_DIR}/fpp-tools/src/compiler/install" "${FPP_TOOLS_PATH}"
TEST_COMMAND ""
)
2 changes: 1 addition & 1 deletion cmake/fpp-download/CMakeLists.sbt.txt.in
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ include(ExternalProject)
ExternalProject_Add(sbt
URL https://github.com/sbt/sbt/releases/download/v1.5.5/sbt-1.5.5.zip
DOWNLOAD_DIR "${CMAKE_BINARY_DIR}/sbt-download"
SOURCE_DIR "${CMAKE_BINARY_DIR}/sbt"
SOURCE_DIR "${FPP_TOOLS_PATH}/sbt"
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
Expand Down
Loading

0 comments on commit 661234d

Please sign in to comment.