Skip to content

Commit

Permalink
lestarch: changes necessary to restrict fpp from regenerating on any …
Browse files Browse the repository at this point in the history
…loc change
  • Loading branch information
LeStarch committed Oct 26, 2021
1 parent dcbe3e3 commit e8349b4
Show file tree
Hide file tree
Showing 6 changed files with 141 additions and 27 deletions.
6 changes: 3 additions & 3 deletions cmake/autocoder/ai-shared.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,11 @@ function(setup_ai_autocode_variant VARIANT_ARGS OUTPUT_DIR EXTRA_COMMANDS AC_INP
BYPRODUCTS ${GENERATED_FILES}
COMMAND ${CMAKE_COMMAND} -E env
PYTHONPATH=${PYTHON_AUTOCODER_DIR}/src:${PYTHON_AUTOCODER_DIR}/utils
BUILD_ROOT="${FPRIME_BUILD_LOCATIONS_SEP}:${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/F-Prime"
FPRIME_AC_CONSTANTS_FILE="${FPRIME_AC_CONSTANTS_FILE}"
BUILD_ROOT=${FPRIME_BUILD_LOCATIONS_SEP}:${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/F-Prime
FPRIME_AC_CONSTANTS_FILE=${FPRIME_AC_CONSTANTS_FILE}
PYTHON_AUTOCODER_DIR=${PYTHON_AUTOCODER_DIR}
${PYTHON} ${FPRIME_FRAMEWORK_PATH}/Autocoders/Python/bin/codegen.py -p ${OUTPUT_DIR} --build_root ${VARIANT_ARGS} ${AC_INPUT_FILE}
COMMAND ${EXTRA_COMMANDS}
DEPENDS ${AC_INPUT_FILE} ${FILE_DEPENDENCIES} ${FPRIME_AC_CONSTANTS_FILE} ${MODULE_DEPENDENCIES} ${CODEGEN_TARGET}
)
endfunction(setup_ai_autocode_variant)
endfunction(setup_ai_autocode_variant)
14 changes: 3 additions & 11 deletions cmake/autocoder/autocoder.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -44,16 +44,8 @@ endfunction()
function (__memoize SOURCES)
string(MD5 SOURCES_HASH "${SOURCES}")
set(MEMO_FILE "${CMAKE_CURRENT_BINARY_DIR}/${AUTOCODER_NAME}.${SOURCES_HASH}.dep")

# Run the expensive action only if the input has changed
on_any_changed("${SOURCES}" CHANGED)
if(COMMAND regenerate_memo)
regenerate_memo(FORCE_REGENERATE "${MEMO_FILE}")
endif()
if (CHANGED OR FORCE_REGENERATE OR NOT EXISTS "${MEMO_FILE}")
if (CMAKE_DEBUG_OUTPUT)
message(STATUS "[Autocode/${AUTOCODER_NAME}] Regenerating memo '${MEMO_FILE}' because: (changed: ${CHANGED}, forced: ${FORCE_REGENERATE})")
endif()
regenerate_memo(FORCE_REGENERATE "${MEMO_FILE}" "${SOURCES}")
if (FORCE_REGENERATE)
get_generated_files("${SOURCES}")
get_dependencies("${SOURCES}")
resolve_dependencies(MODULE_DEPENDENCIES ${MODULE_DEPENDENCIES})
Expand Down Expand Up @@ -138,7 +130,7 @@ function(run_ac AUTOCODER_CMAKE SOURCES GENERATED_SOURCES INFO_ONLY)
endif()
# Configure depends on this source file if it causes a change to module dependencies
if (MODULE_DEPENDENCIES)
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${SOURCES})
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CONSUMED_SOURCES})
endif()

# Return variables
Expand Down
29 changes: 28 additions & 1 deletion cmake/autocoder/default.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,40 @@ function(is_supported AC_INPUT_FILE)
message(FATAL_ERROR "${AUTOCODER_CMAKE} must define 'is_supported'")
endfunction(is_supported)


####
# `default_regenerate_helper`:
#
# Default regenerate helper, so that other regenerates can be built on top of this.
####
function(default_regenerate_helper MISSING_VAR CHANGED_VAR MEMO_FILE SOURCES_INPUT)
# Check for missing memo file
set(MEMO_MISSING TRUE)
if (EXISTS "${MEMO_FILE}")
set(MEMO_MISSING FALSE)
endif()
# Changed sources
on_any_changed("${SOURCES_INPUT}" CHANGED)
set(${MISSING_VAR} ${MEMO_MISSING} PARENT_SCOPE)
set(${CHANGED_VAR} ${CHANGED} PARENT_SCOPE)
endfunction(default_regenerate_helper)


###
# `regenerate_memo`:
#
# Default implementation does not regenerate memo upon request of autocoder.
####
function(regenerate_memo OUTPUT MEMO_FILE)
function(regenerate_memo OUTPUT MEMO_FILE SOURCES_INPUT)
set(${OUTPUT} FALSE PARENT_SCOPE)
default_regenerate_helper(MEMO_MISSING CHANGED "${MEMO_FILE}" "${SOURCES_INPUT}")
# Regenerating on any of the above
if (MEMO_MISSING OR CHANGED)
if (CMAKE_DEBUG_OUTPUT)
message(STATUS "[Autocode/${AUTOCODER_NAME}] Regenerating memo '${MEMO_FILE}' because: (memo missing: ${MEMO_MISSING}, sources changed: ${CHANGED})")
endif()
set(${OUTPUT} TRUE PARENT_SCOPE)
endif()
endfunction(regenerate_memo)

####
Expand Down
58 changes: 58 additions & 0 deletions cmake/autocoder/fpp-locs-differ/fpp-locs-differ.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import os
import sys
import argparse
import difflib
from fprime_ac.utils.buildroot import (
search_for_file,
build_root_relative_path,
set_build_roots,
)


def main():
"""Process arguments: file old, file new, module deps to see if any mod deps has changed"""
set_build_roots(os.environ["BUILD_ROOT"])
parser = argparse.ArgumentParser(
description="Has a module dependency changed within the locs file."
)
parser.add_argument("new_locs", help="New locs file")
parser.add_argument("prev_locs", help="Original locs file")
parser.add_argument("file_deps", nargs="+", help="Module deps")

args_ns = parser.parse_args()

if not os.path.exists(args_ns.new_locs):
print(f"[ERROR] Failed to open {args_ns.new_locs}")
sys.exit(1)
# Previous files not generated
if not os.path.exists(args_ns.prev_locs):
print(f"No pervious locations")
sys.exit(1)

with open(args_ns.prev_locs, "r") as prev_locs_fh:
prev_lines = prev_locs_fh.readlines()
with open(args_ns.new_locs, "r") as new_locs_fh:
new_lines = new_locs_fh.readlines()

diff_output = list(difflib.unified_diff(prev_lines, new_lines, n=0, lineterm="\n"))
subtracted_lines = [diff for diff in diff_output if diff.startswith("-") and not diff.startswith("---")]
relative_mod_paths = [line.split()[-1].strip('"') for line in subtracted_lines]
subtractions = [
os.path.abspath(
os.path.join(
os.path.dirname(args_ns.new_locs), rel_path)
)
for rel_path in relative_mod_paths
]
changed = []
for subtraction in subtractions:
if subtraction in args_ns.file_deps:
changed.append(os.path.basename(subtraction))
if changed:
print(f"{', '.join(changed)}")
sys.exit(2)
sys.exit(0)


if __name__ == "__main__":
main()
44 changes: 39 additions & 5 deletions cmake/autocoder/fpp.cmake
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
include(utilities)
include(autocoder/default)

set(HANDLES_INDIVIDUAL_SOURCES FALSE)

Expand All @@ -10,11 +11,43 @@ function(is_supported AC_INPUT_FILE)
endif()
endfunction(is_supported)

#TODO: add a correction function w.r.t. removing items from topologies
function(regenerate_memo OUTPUT MEMO_FILE SOURCES_INPUT)
# Initialize variables
set(${OUTPUT} FALSE PARENT_SCOPE)
set(LOCS_MOVED FALSE)
set(OUTPUT_TEXT "n/a")
set(ALL_FPP_INPUTS)
set(PREV_LOCS_FILE "${MEMO_FILE}.locs.prev")
default_regenerate_helper(MEMO_MISSING CHANGED "${MEMO_FILE}" "${SOURCES_INPUT}")

function(regenerate_memo OUTPUT MEMO_FILE)
on_any_changed("${FPP_LOCS_FILE}" CHANGED) #TODO: fix this function
set(${OUTPUT} "${CHANGED}" PARENT_SCOPE)
# Read the memo file for dependencies, should it exist
if (NOT MEMO_MISSING)
file(READ "${MEMO_FILE}" CONTENTS)
read_from_lines("${CONTENTS}" GENERATED_FILES MODULE_DEPENDENCIES FILE_DEPENDENCIES ALL_FPP_INPUTS)
endif()

# Look at changed inter-module dependencies
if (ALL_FPP_INPUTS)
# Subset module deps by what changed
string(REPLACE ";" ":" FPRIME_BUILD_LOCATIONS_SEP "${FPRIME_BUILD_LOCATIONS}")
find_program(PYTHON NAMES python3 python REQUIRED)
execute_process(COMMAND ${CMAKE_COMMAND} -E env
PYTHONPATH=${PYTHON_AUTOCODER_DIR}/src:${PYTHON_AUTOCODER_DIR}/utils
BUILD_ROOT=${FPRIME_BUILD_LOCATIONS_SEP}:${CMAKE_BINARY_DIR}:${CMAKE_BINARY_DIR}/F-Prime
${PYTHON} "${FPRIME_FRAMEWORK_PATH}/cmake/autocoder/fpp-locs-differ/fpp-locs-differ.py" ${FPP_LOCS_FILE} ${PREV_LOCS_FILE}
${ALL_FPP_INPUTS} OUTPUT_VARIABLE OUTPUT_TEXT OUTPUT_STRIP_TRAILING_WHITESPACE RESULT_VARIABLE RESULT_OUT)
if (NOT RESULT_OUT EQUAL "0")
set(LOCS_MOVED TRUE)
endif()
endif()
# Regenerating on any of the above
if (MEMO_MISSING OR CHANGED OR LOCS_MOVED)
execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different ${FPP_LOCS_FILE} ${PREV_LOCS_FILE})
if (CMAKE_DEBUG_OUTPUT)
message(STATUS "[Autocode/${AUTOCODER_NAME}] Regenerating memo '${MEMO_FILE}' because: (memo missing: ${MEMO_MISSING}, sources changed: ${CHANGED}, dependencies moved: ${LOCS_MOVED} (${OUTPUT_TEXT}))")
endif()
set(${OUTPUT} TRUE PARENT_SCOPE)
endif()
endfunction(regenerate_memo)

function(get_generated_files AC_INPUT_FILES)
Expand All @@ -26,7 +59,8 @@ function(get_generated_files AC_INPUT_FILES)
set(INCLUDED_FILE "${CMAKE_CURRENT_BINARY_DIR}/included.txt")
set(MISSING_FILE "${CMAKE_CURRENT_BINARY_DIR}/missing.txt")
set(GENERATED_FILE "${CMAKE_CURRENT_BINARY_DIR}/generated.txt")
set(LAST_DEP_COMMAND "${FPP_DEPEND} ${FPP_LOCS_FILE} ${AC_INPUT_FILES} -d ${DIRECT_FILE} -i ${INCLUDED_FILE} -m ${MISSING_FILE} -g ${GENERATED_FILE}" CACHE INTERNAL "Last command to annotate memo file" FORCE)
set(LAST_DEP_COMMAND "${FPP_DEPEND} ${FPP_LOCS_FILE} ${AC_INPUT_FILES} -d ${DIRECT_FILE} -i ${INCLUDED_FILE} -m ${MISSING_FILE} -g ${GENERATED_FILE}"
CACHE INTERNAL "Last command to annotate memo file" FORCE)
execute_process(COMMAND ${FPP_DEPEND} ${FPP_LOCS_FILE} ${AC_INPUT_FILES}
-d "${DIRECT_FILE}"
-i "${INCLUDED_FILE}"
Expand Down
17 changes: 10 additions & 7 deletions cmake/target/fpp-locs.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,16 @@ function(generate_fpp_locs)
set(LOCS_DIR "${CMAKE_BINARY_DIR}/fpp-locs")
set(LOCS_FILE "${LOCS_DIR}/locs.fpp")

file(MAKE_DIRECTORY "${LOCS_DIR}")
execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" "${CMAKE_CURRENT_SOURCE_DIR}" -DFPRIME_FPP_LOCS_BUILD=ON ${CALL_PROPS}
RESULT_VARIABLE result
OUTPUT_FILE "${LOCS_DIR}/generate-output.log"
WORKING_DIRECTORY "${LOCS_DIR}")
if(result)
message(FATAL_ERROR "CMake step for generating fpp-locs build failed: ${result}")
# Only generate the FPP locs cache when it doesn't exist. Otherwise just build.
if (NOT IS_DIRECTORY LOCS_DIR)
file(MAKE_DIRECTORY "${LOCS_DIR}")
execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" "${CMAKE_CURRENT_SOURCE_DIR}" -DFPRIME_FPP_LOCS_BUILD=ON ${CALL_PROPS}
RESULT_VARIABLE result
OUTPUT_FILE "${LOCS_DIR}/generate-output.log"
WORKING_DIRECTORY "${LOCS_DIR}")
if(result)
message(FATAL_ERROR "CMake step for generating fpp-locs build failed: ${result}")
endif()
endif()
execute_process(COMMAND ${CMAKE_COMMAND} --build . --target fpp-locs
RESULT_VARIABLE result
Expand Down

0 comments on commit e8349b4

Please sign in to comment.