diff --git a/CMakeLists.txt b/CMakeLists.txt index 5bd9348cb57e88..e994ddfa87b26d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1990,7 +1990,6 @@ if (CONFIG_LLEXT AND CONFIG_LLEXT_EXPORT_BUILTINS_BY_SLID) --elf-file ${PROJECT_BINARY_DIR}/${KERNEL_ELF_NAME} --slid-listing ${PROJECT_BINARY_DIR}/slid_listing.txt ) - endif() if(NOT CMAKE_C_COMPILER_ID STREQUAL "ARMClang") @@ -2236,6 +2235,10 @@ list(APPEND llext_edk_cflags ${llext_filt_flags}) list(APPEND llext_edk_cflags ${LLEXT_APPEND_FLAGS}) list(APPEND llext_edk_cflags ${LLEXT_EDK_APPEND_FLAGS}) +build_info(llext-edk file PATH ${llext_edk_file}) +build_info(llext-edk cflags VALUE ${llext_edk_cflags} GENEX) +build_info(llext-edk include-dirs VALUE "$" GENEX) + add_custom_command( OUTPUT ${llext_edk_file} # Regenerate syscalls in case CONFIG_LLEXT_EDK_USERSPACE_ONLY @@ -2252,15 +2255,6 @@ add_custom_command( ${SYSCALL_LONG_REGISTERS_ARG} ${SYSCALL_SPLIT_TIMEOUT_ARG} COMMAND ${CMAKE_COMMAND} - -DPROJECT_BINARY_DIR=${PROJECT_BINARY_DIR} - -DAPPLICATION_SOURCE_DIR=${APPLICATION_SOURCE_DIR} - -DINTERFACE_INCLUDE_DIRECTORIES="$" - -Dllext_edk_file=${llext_edk_file} - -Dllext_edk_cflags="${llext_edk_cflags}" - -Dllext_edk_name=${CONFIG_LLEXT_EDK_NAME} - -DWEST_TOPDIR=${WEST_TOPDIR} - -DZEPHYR_BASE=${ZEPHYR_BASE} - -DCONFIG_LLEXT_EDK_USERSPACE_ONLY=${CONFIG_LLEXT_EDK_USERSPACE_ONLY} -P ${ZEPHYR_BASE}/cmake/llext-edk.cmake DEPENDS ${logical_target_for_zephyr_elf} COMMAND_EXPAND_LISTS @@ -2286,9 +2280,8 @@ add_subdirectory_ifdef( toolchain_linker_finalize() -yaml_context(EXISTS NAME build_info result) -if(result) - build_info(zephyr version VALUE ${PROJECT_VERSION_STR}) - build_info(zephyr zephyr-base VALUE ${ZEPHYR_BASE}) - yaml_save(NAME build_info) -endif() +# export build information +build_info(zephyr version VALUE ${PROJECT_VERSION_STR}) +build_info(zephyr zephyr-base VALUE ${ZEPHYR_BASE}) + +yaml_save(NAME build_info TARGET ${logical_target_for_zephyr_elf}) diff --git a/cmake/llext-edk.cmake b/cmake/llext-edk.cmake index e78498c7845250..6bca08fc48f9e1 100644 --- a/cmake/llext-edk.cmake +++ b/cmake/llext-edk.cmake @@ -11,33 +11,31 @@ # directories (build/zephyr, zephyr base, west top dir and application source # dir), to avoid leaking any information about the host system. # -# The following arguments are expected: -# - llext_edk_name: Name of the extension, used to name the tarball and the -# install directory variable for Makefile. -# - INTERFACE_INCLUDE_DIRECTORIES: List of include directories to copy headers -# from. It should simply be the INTERFACE_INCLUDE_DIRECTORIES property of the -# zephyr_interface target. -# - llext_edk_file: Output file name for the tarball. -# - llext_edk_cflags: Flags to be used for source compile commands. -# - ZEPHYR_BASE: Path to the zephyr base directory. -# - WEST_TOPDIR: Path to the west top directory. -# - APPLICATION_SOURCE_DIR: Path to the application source directory. -# - PROJECT_BINARY_DIR: Path to the project binary build directory. -# - CONFIG_LLEXT_EDK_USERSPACE_ONLY: Whether to copy syscall headers from the -# edk directory. This is necessary when building an extension that only -# supports userspace, as the syscall headers are regenerated in the edk -# directory. +# The script expects a build_info.yml file in the project binary directory. +# This file should contain the following entries: +# - cmake application source-dir +# - cmake llext-edk cflags +# - cmake llext-edk file +# - cmake llext-edk include-dirs +# - west topdir cmake_minimum_required(VERSION 3.20.0) +# initialize the same paths as the main CMakeLists.txt for consistency +set(PROJECT_BINARY_DIR ${CMAKE_BINARY_DIR}) +set(ZEPHYR_BASE ${CMAKE_CURRENT_LIST_DIR}/../) +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/modules") +include(extensions) +include(yaml) + +# read in computed build configuration +import_kconfig(CONFIG ${PROJECT_BINARY_DIR}/.config) + if (CONFIG_LLEXT_EXPORT_BUILTINS_BY_SLID) message(FATAL_ERROR "The LLEXT EDK is not compatible with CONFIG_LLEXT_EXPORT_BUILTINS_BY_SLID.") endif() -set(llext_edk ${PROJECT_BINARY_DIR}/${llext_edk_name}) -set(llext_edk_inc ${llext_edk}/include) - # Usage: # relative_dir( ) # @@ -89,12 +87,23 @@ function(relative_dir dir relative_out bindir_out) endif() endfunction() +set(build_info_file ${PROJECT_BINARY_DIR}/../build_info.yml) +yaml_load(FILE ${build_info_file} NAME build_info) + +yaml_get(llext_edk_cflags NAME build_info KEY cmake llext-edk cflags) +yaml_get(llext_edk_file NAME build_info KEY cmake llext-edk file) +yaml_get(INTERFACE_INCLUDE_DIRECTORIES NAME build_info KEY cmake llext-edk include-dirs) +yaml_get(APPLICATION_SOURCE_DIR NAME build_info KEY cmake application source-dir) +yaml_get(WEST_TOPDIR NAME build_info KEY west topdir) + +set(llext_edk_name ${CONFIG_LLEXT_EDK_NAME}) +set(llext_edk ${PROJECT_BINARY_DIR}/${llext_edk_name}) +set(llext_edk_inc ${llext_edk}/include) + string(REGEX REPLACE "[^a-zA-Z0-9]" "_" llext_edk_name_sane ${llext_edk_name}) string(TOUPPER ${llext_edk_name_sane} llext_edk_name_sane) set(install_dir_var "${llext_edk_name_sane}_INSTALL_DIR") -separate_arguments(llext_edk_cflags NATIVE_COMMAND ${llext_edk_cflags}) - set(make_relative FALSE) foreach(flag ${llext_edk_cflags}) if (flag STREQUAL "-imacros") @@ -106,11 +115,9 @@ foreach(flag ${llext_edk_cflags}) relative_dir(${parent} dest bindir) cmake_path(RELATIVE_PATH dest BASE_DIRECTORY ${llext_edk} OUTPUT_VARIABLE dest_rel) if(bindir) - list(APPEND imacros_gen_make "-imacros\$(${install_dir_var})/${dest_rel}/${name}") - list(APPEND imacros_gen_cmake "-imacros\${CMAKE_CURRENT_LIST_DIR}/${dest_rel}/${name}") + list(APPEND imacros_gen "@DASHIMACROS@${dest_rel}/${name}") else() - list(APPEND imacros_make "-imacros\$(${install_dir_var})/${dest_rel}/${name}") - list(APPEND imacros_cmake "-imacros\${CMAKE_CURRENT_LIST_DIR}/${dest_rel}/${name}") + list(APPEND imacros "@DASHIMACROS@${dest_rel}/${name}") endif() else() list(APPEND new_cflags ${flag}) @@ -118,12 +125,10 @@ foreach(flag ${llext_edk_cflags}) endforeach() set(llext_edk_cflags ${new_cflags}) -list(APPEND base_flags_make ${llext_edk_cflags} ${imacros_make}) -list(APPEND base_flags_cmake ${llext_edk_cflags} ${imacros_cmake}) +list(APPEND base_flags ${llext_edk_cflags} ${imacros}) -separate_arguments(include_dirs NATIVE_COMMAND ${INTERFACE_INCLUDE_DIRECTORIES}) file(MAKE_DIRECTORY ${llext_edk_inc}) -foreach(dir ${include_dirs}) +foreach(dir ${INTERFACE_INCLUDE_DIRECTORIES}) if (NOT EXISTS ${dir}) continue() endif() @@ -137,54 +142,85 @@ foreach(dir ${include_dirs}) cmake_path(RELATIVE_PATH dest BASE_DIRECTORY ${llext_edk} OUTPUT_VARIABLE dest_rel) if(bindir) - list(APPEND inc_gen_flags_make "-I\$(${install_dir_var})/${dest_rel}") - list(APPEND inc_gen_flags_cmake "-I\${CMAKE_CURRENT_LIST_DIR}/${dest_rel}") + list(APPEND gen_inc_flags "@DASHI@${dest_rel}") else() - list(APPEND inc_flags_make "-I\$(${install_dir_var})/${dest_rel}") - list(APPEND inc_flags_cmake "-I\${CMAKE_CURRENT_LIST_DIR}/${dest_rel}") + list(APPEND inc_flags "@DASHI@${dest_rel}") endif() - list(APPEND all_inc_flags_make "-I\$(${install_dir_var})/${dest_rel}") - list(APPEND all_inc_flags_cmake "-I\${CMAKE_CURRENT_LIST_DIR}/${dest_rel}") + list(APPEND all_inc_flags "@DASHI@${dest_rel}") endforeach() +list(APPEND all_flags ${base_flags} ${imacros_gen} ${all_inc_flags}) + if(CONFIG_LLEXT_EDK_USERSPACE_ONLY) # Copy syscall headers from edk directory, as they were regenerated there. file(COPY ${PROJECT_BINARY_DIR}/edk/include/generated/ DESTINATION ${llext_edk_inc}/zephyr/include/generated) endif() -# Generate flags for Makefile -list(APPEND all_flags_make ${base_flags_make} ${imacros_gen_make} ${all_inc_flags_make}) -list(JOIN all_flags_make " " all_flags_str) -file(WRITE ${llext_edk}/Makefile.cflags "LLEXT_CFLAGS = ${all_flags_str}") -list(JOIN all_inc_flags_make " " all_inc_flags_str) -file(APPEND ${llext_edk}/Makefile.cflags "\n\nLLEXT_ALL_INCLUDE_CFLAGS = ${all_inc_flags_str}") - -list(JOIN inc_flags_make " " inc_flags_str) -file(APPEND ${llext_edk}/Makefile.cflags "\n\nLLEXT_INCLUDE_CFLAGS = ${inc_flags_str}") - -list(JOIN inc_gen_flags_make " " inc_gen_flags_str) -file(APPEND ${llext_edk}/Makefile.cflags "\n\nLLEXT_GENERATED_INCLUDE_CFLAGS = ${inc_gen_flags_str}") - -list(JOIN base_flags_make " " base_flags_str) -file(APPEND ${llext_edk}/Makefile.cflags "\n\nLLEXT_BASE_CFLAGS = ${base_flags_str}") +# +# Generate the EDK flags files +# -list(JOIN imacros_gen_make " " imacros_gen_str) -file(APPEND ${llext_edk}/Makefile.cflags "\n\nLLEXT_GENERATED_IMACROS_CFLAGS = ${imacros_gen_str}") +set(edk_targets MAKEFILE CMAKE) +set(edk_file_MAKEFILE ${llext_edk}/Makefile.cflags) +set(edk_file_CMAKE ${llext_edk}/cmake.cflags) -# Generate flags for CMake -list(APPEND all_flags_cmake ${base_flags_cmake} ${imacros_gen_cmake} ${all_inc_flags_cmake}) -file(WRITE ${llext_edk}/cmake.cflags "set(LLEXT_CFLAGS ${all_flags_cmake})") +# Escape problematic characters in a string +function(edk_escape target str_in str_out) + string(REPLACE "\\" "\\\\" str_escaped "${str_in}") + string(REPLACE "\"" "\\\"" str_escaped "${str_escaped}") + set(${str_out} "${str_escaped}" PARENT_SCOPE) +endfunction() -file(APPEND ${llext_edk}/cmake.cflags "\n\nset(LLEXT_ALL_INCLUDE_CFLAGS ${all_inc_flags_cmake})") +# Clear the contents of the requested file +function(edk_write_header target) + file(WRITE ${edk_file_${target}} "") +endfunction() -file(APPEND ${llext_edk}/cmake.cflags "\n\nset(LLEXT_INCLUDE_CFLAGS ${inc_flags_cmake})") +# Mark a section in the file with a single line comment +function(edk_write_comment target comment) + file(APPEND ${edk_file_${target}} "\n# ${comment}\n") +endfunction() -file(APPEND ${llext_edk}/cmake.cflags "\n\nset(LLEXT_GENERATED_INCLUDE_CFLAGS ${inc_gen_flags_cmake})") +# Define a variable in the file +function(edk_write_var target var_name var_value) + if(target STREQUAL "CMAKE") + # CMake: export assignments of the form: + # + # set(var "value1;value2;...") + # + set(DASHIMACROS "-imacros\${CMAKE_CURRENT_LIST_DIR}/") + set(DASHI "-I\${CMAKE_CURRENT_LIST_DIR}/") + edk_escape(${target} "${var_value}" var_value) + string(CONFIGURE "${var_value}" exp_var_value @ONLY) + # The list is otherwise exported verbatim, surrounded by quotes. + file(APPEND ${edk_file_${target}} "set(${var_name} \"${exp_var_value}\")\n") + elseif(target STREQUAL "MAKEFILE") + # Makefile: export assignments of the form: + # + # var = "value1" "value2" ... + # + set(DASHIMACROS "-imacros\$(${install_dir_var})/") + set(DASHI "-I\$(${install_dir_var})/") + edk_escape(${target} "${var_value}" var_value) + string(CONFIGURE "${var_value}" exp_var_value @ONLY) + # Each element of the list is wrapped in quotes and is separated by a space. + list(JOIN exp_var_value "\" \"" exp_var_value_str) + file(APPEND ${edk_file_${target}} "${var_name} = \"${exp_var_value_str}\"\n") + endif() +endfunction() -file(APPEND ${llext_edk}/cmake.cflags "\n\nset(LLEXT_BASE_CFLAGS ${base_flags_cmake})") +foreach(target ${edk_targets}) + edk_write_header(${target}) -file(APPEND ${llext_edk}/cmake.cflags "\n\nset(LLEXT_GENERATED_IMACROS_CFLAGS ${imacros_gen_cmake})") + edk_write_comment(${target} "Compile flags") + edk_write_var(${target} "LLEXT_CFLAGS" "${all_flags}") + edk_write_var(${target} "LLEXT_ALL_INCLUDE_CFLAGS" "${all_inc_flags}") + edk_write_var(${target} "LLEXT_INCLUDE_CFLAGS" "${inc_flags}") + edk_write_var(${target} "LLEXT_GENERATED_INCLUDE_CFLAGS" "${gen_inc_flags}") + edk_write_var(${target} "LLEXT_BASE_CFLAGS" "${base_flags}") + edk_write_var(${target} "LLEXT_GENERATED_IMACROS_CFLAGS" "${imacros_gen}") +endforeach() # Generate the tarball file(ARCHIVE_CREATE diff --git a/cmake/modules/extensions.cmake b/cmake/modules/extensions.cmake index 411398b0ae8dbc..7823f7a0f21ab9 100644 --- a/cmake/modules/extensions.cmake +++ b/cmake/modules/extensions.cmake @@ -3703,10 +3703,10 @@ function(topological_sort) endfunction() # Usage: -# build_info(... VALUE ... ) +# build_info(... VALUE ... [GENEX]) # build_info(... PATH ... ) # -# This function populates updates the build_info.yml info file with exchangable build information +# This function populates the build_info.yml info file with exchangable build information # related to the current build. # # Example: @@ -3724,17 +3724,27 @@ endfunction() # PATH ... : path(s) to place in the build_info.yml file. All paths are converted to CMake # style. If no conversion is required, for example when paths are already # guaranteed to be CMake style, then VALUE can also be used. +# GENEX : the value(s) contain a generator expression. Cannot be used with PATH. +# Resulting context must be saved providing a TARGET to yaml_save(). function(build_info) - set(convert_path FALSE) - set(arg_list ${ARGV}) - list(FIND arg_list VALUE index) - if(index EQUAL -1) - list(FIND arg_list PATH index) + cmake_parse_arguments(ARG_BUILD_INFO "GENEX" "" "VALUE;PATH" ${ARGN}) + + zephyr_check_arguments_required_allow_empty(${CMAKE_CURRENT_FUNCTION} ARG_BUILD_INFO VALUE PATH) + zephyr_check_arguments_exclusive(${CMAKE_CURRENT_FUNCTION} ARG_BUILD_INFO VALUE PATH) + + set(keys ${ARG_BUILD_INFO_UNPARSED_ARGUMENTS}) + if (DEFINED ARG_BUILD_INFO_PATH) set(convert_path TRUE) + set(values ${ARG_BUILD_INFO_PATH}) + else() + set(convert_path FALSE) + set(values ${ARG_BUILD_INFO_VALUE}) endif() - - if(index EQUAL -1) - message(FATAL_ERROR "${CMAKE_CURRENT_FUNCTION}(...) missing a required argument: VALUE or PATH") + if (ARG_BUILD_INFO_GENEX) + if (convert_path) + message(FATAL_ERROR "build_info: GENEX is unsupported on PATH entries") + endif() + set(genex_flag "GENEX") endif() yaml_context(EXISTS NAME build_info result) @@ -3748,10 +3758,6 @@ function(build_info) yaml_set(NAME build_info KEY version VALUE "0.1.0") endif() - list(SUBLIST arg_list 0 ${index} keys) - list(SUBLIST arg_list ${index} -1 values) - list(POP_FRONT values) - if(convert_path) set(converted_values) foreach(val ${values}) @@ -3779,7 +3785,7 @@ function(build_info) endif() endif() - yaml_set(NAME build_info KEY cmake ${keys} ${type} "${values}") + yaml_set(NAME build_info KEY cmake ${keys} ${type} "${values}" ${genex_flag}) endfunction() ######################################################## @@ -5916,7 +5922,7 @@ endfunction() # depending on the exact use of the function in script mode. # # Current Zephyr CMake scripts which includes `extensions.cmake` in script mode -# are: package_helper.cmake, verify-toolchain.cmake +# are: package_helper.cmake, verify-toolchain.cmake, llext-edk.cmake # if(CMAKE_SCRIPT_MODE_FILE) diff --git a/cmake/modules/yaml.cmake b/cmake/modules/yaml.cmake index 9ab9b333fa46cc..8c57d55f5d169a 100644 --- a/cmake/modules/yaml.cmake +++ b/cmake/modules/yaml.cmake @@ -72,6 +72,54 @@ function(internal_yaml_context_free) endif() endfunction() +# Internal helper function to provide the correct initializer for a list in the +# JSON content. +function(internal_yaml_list_initializer var genex) + if(genex) + set(${var} "\"@YAML-LIST@\"" PARENT_SCOPE) + else() + set(${var} "[]" PARENT_SCOPE) + endif() +endfunction() + +# Internal helper function to append items to a list in the JSON content. +# Unassigned arguments are the values to be appended. +function(internal_yaml_list_append var genex key) + set(json_content "${${var}}") + string(JSON subjson GET "${json_content}" ${key}) + if(genex) + # new lists are stored in CMake string format, but those imported via + # yaml_load() are proper JSON arrays. When an append is requested, those + # must be converted back to a CMake list. + string(JSON type TYPE "${json_content}" ${key}) + if(type STREQUAL ARRAY) + string(JSON arraylength LENGTH "${subjson}") + internal_yaml_list_initializer(subjson TRUE) + if(${arraylength} GREATER 0) + math(EXPR arraystop "${arraylength} - 1") + foreach(i RANGE 0 ${arraystop}) + string(JSON item GET "${json_content}" ${key} ${i}) + list(APPEND subjson ${item}) + endforeach() + endif() + endif() + list(APPEND subjson ${ARGN}) + string(JSON json_content SET "${json_content}" ${key} "\"${subjson}\"") + else() + # lists are stored as JSON arrays + string(JSON index LENGTH "${subjson}") + list(LENGTH ARGN length) + math(EXPR stop "${index} + ${length} - 1") + if(NOT length EQUAL 0) + foreach(i RANGE ${index} ${stop}) + list(POP_FRONT ARGN value) + string(JSON json_content SET "${json_content}" ${key} ${i} "\"${value}\"") + endforeach() + endif() + endif() + set(${var} "${json_content}" PARENT_SCOPE) +endfunction() + # Usage # yaml_context(EXISTS NAME ) # @@ -125,6 +173,7 @@ function(yaml_create) if(DEFINED ARG_YAML_FILE) zephyr_set(FILE ${ARG_YAML_FILE} SCOPE ${ARG_YAML_NAME}) endif() + zephyr_set(GENEX FALSE SCOPE ${ARG_YAML_NAME}) zephyr_set(JSON "{}" SCOPE ${ARG_YAML_NAME}) endfunction() @@ -149,7 +198,7 @@ function(yaml_load) zephyr_set(FILE ${ARG_YAML_FILE} SCOPE ${ARG_YAML_NAME}) execute_process(COMMAND ${PYTHON_EXECUTABLE} -c - "import json; import yaml; print(json.dumps(yaml.safe_load(open('${ARG_YAML_FILE}'))))" + "import json; import yaml; print(json.dumps(yaml.safe_load(open('${ARG_YAML_FILE}')) or {}))" OUTPUT_VARIABLE json_load_out ERROR_VARIABLE json_load_error RESULT_VARIABLE json_load_result @@ -161,6 +210,7 @@ function(yaml_load) ) endif() + zephyr_set(GENEX FALSE SCOPE ${ARG_YAML_NAME}) zephyr_set(JSON "${json_load_out}" SCOPE ${ARG_YAML_NAME}) endfunction() @@ -241,8 +291,8 @@ function(yaml_length out_var) endfunction() # Usage: -# yaml_set(NAME KEY ... VALUE ) -# yaml_set(NAME KEY ... [APPEND] LIST ...) +# yaml_set(NAME KEY ... [GENEX] VALUE ) +# yaml_set(NAME KEY ... [APPEND] [GENEX] LIST ...) # # Set a value or a list of values to given key. # @@ -252,18 +302,32 @@ endfunction() # NAME : Name of the YAML context. # KEY ... : Name of key. # VALUE : New value for the key. -# List : New list of values for the key. +# LIST : New list of values for the key. # APPEND : Append the list of values to the list of values for the key. +# GENEX : The value(s) contain generator expressions. The YAML context +# has to be written passing a TARGET to 'yaml_save()' for these +# to be properly expanded. # function(yaml_set) - cmake_parse_arguments(ARG_YAML "APPEND" "NAME;VALUE" "KEY;LIST" ${ARGN}) + cmake_parse_arguments(ARG_YAML "APPEND;GENEX" "NAME;VALUE" "KEY;LIST" ${ARGN}) zephyr_check_arguments_required_all(${CMAKE_CURRENT_FUNCTION} ARG_YAML NAME KEY) zephyr_check_arguments_required_allow_empty(${CMAKE_CURRENT_FUNCTION} ARG_YAML VALUE LIST) zephyr_check_arguments_exclusive(${CMAKE_CURRENT_FUNCTION} ARG_YAML VALUE LIST) internal_yaml_context_required(NAME ${ARG_YAML_NAME}) + if(ARG_YAML_GENEX) + zephyr_set(GENEX TRUE SCOPE ${ARG_YAML_NAME}) + endif() + zephyr_get_scoped(json_content ${ARG_YAML_NAME} JSON) + zephyr_get_scoped(genex ${ARG_YAML_NAME} GENEX) + + if(DEFINED ARG_YAML_LIST + OR LIST IN_LIST ARG_YAML_KEYWORDS_MISSING_VALUES + OR ARG_YAML_APPEND) + set(key_is_list TRUE) + endif() set(yaml_key_undefined ${ARG_YAML_KEY}) foreach(k ${yaml_key_undefined}) @@ -281,8 +345,8 @@ function(yaml_set) list(REVERSE yaml_key_undefined) if(NOT "${yaml_key_undefined}" STREQUAL "") - if(ARG_YAML_APPEND) - set(json_string "[]") + if(key_is_list) + internal_yaml_list_initializer(json_string ${genex}) else() set(json_string "\"\"") endif() @@ -295,21 +359,13 @@ function(yaml_set) ) endif() - if(DEFINED ARG_YAML_LIST OR LIST IN_LIST ARG_YAML_KEYWORDS_MISSING_VALUES) + if(key_is_list) if(NOT ARG_YAML_APPEND) - string(JSON json_content SET "${json_content}" ${ARG_YAML_KEY} "[]") + internal_yaml_list_initializer(json_string ${genex}) + string(JSON json_content SET "${json_content}" ${ARG_YAML_KEY} "${json_string}") endif() - string(JSON subjson GET "${json_content}" ${ARG_YAML_KEY}) - string(JSON index LENGTH "${subjson}") - list(LENGTH ARG_YAML_LIST length) - math(EXPR stop "${index} + ${length} - 1") - if(NOT length EQUAL 0) - foreach(i RANGE ${index} ${stop}) - list(POP_FRONT ARG_YAML_LIST value) - string(JSON json_content SET "${json_content}" ${ARG_YAML_KEY} ${i} "\"${value}\"") - endforeach() - endif() + internal_yaml_list_append(json_content ${genex} "${ARG_YAML_KEY}" ${ARG_YAML_LIST}) else() string(JSON json_content SET "${json_content}" ${ARG_YAML_KEY} "\"${ARG_YAML_VALUE}\"") endif() @@ -343,19 +399,25 @@ function(yaml_remove) endfunction() # Usage: -# yaml_save(NAME [FILE ]) +# yaml_save(NAME [TARGET ] [FILE ]) # -# Write the YAML context to the file which were given with the earlier -# 'yaml_load()' or 'yaml_create()' call. +# Write the YAML context to , or the one given with the earlier +# 'yaml_load()' or 'yaml_create()' call. This will be performed immediately if +# the context does not use generator expressions; otherwise, keys that include a +# generator expression will initially be written as comments, and the full contents +# will be available at build time as a pre-build step of . # # NAME : Name of the YAML context # FILE : Path to file to write the context. # If not given, then the FILE property of the YAML context will be # used. In case both FILE is omitted and FILE property is missing # on the YAML context, then an error will be raised. +# TARGET : Target to be used as reference for generator expansions. +# Must be provided to write the expanded contents at build time, as +# a pre-build step of . # function(yaml_save) - cmake_parse_arguments(ARG_YAML "" "NAME;FILE" "" ${ARGN}) + cmake_parse_arguments(ARG_YAML "" "NAME;FILE;TARGET" "" ${ARGN}) zephyr_check_arguments_required(${CMAKE_CURRENT_FUNCTION} ARG_YAML NAME) internal_yaml_context_required(NAME ${ARG_YAML_NAME}) @@ -364,22 +426,50 @@ function(yaml_save) if(NOT yaml_file) zephyr_check_arguments_required(${CMAKE_CURRENT_FUNCTION} ARG_YAML FILE) endif() - - zephyr_get_scoped(json_content ${ARG_YAML_NAME} JSON) - to_yaml("${json_content}" 0 yaml_out) - if(DEFINED ARG_YAML_FILE) set(yaml_file ${ARG_YAML_FILE}) else() zephyr_get_scoped(yaml_file ${ARG_YAML_NAME} FILE) endif() + + zephyr_get_scoped(genex ${ARG_YAML_NAME} GENEX) + if(genex AND NOT ARG_YAML_TARGET) + message(WARNING "YAML context '${ARG_YAML_NAME}' uses generator expressions " + "which require a TARGET parameter to 'yaml_save()'. GENEX " + "entries will be included as comments in the output." + ) + endif() + + zephyr_get_scoped(json_content ${ARG_YAML_NAME} JSON) + to_yaml("${json_content}" 0 yaml_out ${genex}) + if(EXISTS ${yaml_file}) FILE(RENAME ${yaml_file} ${yaml_file}.bak) endif() FILE(WRITE ${yaml_file} "${yaml_out}") + + if (genex AND ARG_YAML_TARGET) + cmake_path(SET yaml_path "${yaml_file}") + cmake_path(GET yaml_path STEM yaml_file_no_ext) + set(json_file ${yaml_file_no_ext}.tmp) + + FILE(GENERATE OUTPUT ${json_file} + CONTENT "${json_content}" + TARGET ${ARG_YAML_TARGET} + ) + + add_custom_command(TARGET ${ARG_YAML_TARGET} PRE_BUILD + BYPRODUCTS ${yaml_file} + COMMAND ${CMAKE_COMMAND} + -DJSON_FILE=${json_file} + -DYAML_FILE=${yaml_file} + -P ${ZEPHYR_BASE}/cmake/yaml-filter.cmake + ) + add_custom_target(${ARG_YAML_NAME}_yaml_filter ALL DEPENDS ${yaml_file}) + endif() endfunction() -function(to_yaml json level yaml) +function(to_yaml json level yaml genex) if(level GREATER 0) math(EXPR level_dec "${level} - 1") set(indent_${level} "${indent_${level_dec}} ") @@ -398,10 +488,12 @@ function(to_yaml json level yaml) string(JSON type TYPE "${json}" ${member}) string(JSON subjson GET "${json}" ${member}) if(type STREQUAL OBJECT) + #-------------------- set(${yaml} "${${yaml}}${indent_${level}}${member}:\n") math(EXPR sublevel "${level} + 1") - to_yaml("${subjson}" ${sublevel} ${yaml}) + to_yaml("${subjson}" ${sublevel} ${yaml} ${genex}) elseif(type STREQUAL ARRAY) + # ------------------- set(${yaml} "${${yaml}}${indent_${level}}${member}:") string(JSON arraylength LENGTH "${subjson}") if(${arraylength} LESS 1) @@ -414,7 +506,30 @@ function(to_yaml json level yaml) set(${yaml} "${${yaml}}${indent_${level}} - ${item}\n") endforeach() endif() + elseif(type STREQUAL STRING) + # -------------------- + if (subjson MATCHES "\\$<.*>" AND ${genex}) + # Yet unexpanded generator expression: save as comment + string(SUBSTRING ${indent_${level}} 1 -1 short_indent) + set(${yaml} "${${yaml}}#${short_indent}${member}: ${subjson}\n") + elseif(subjson MATCHES "^@YAML-LIST@") + # List-as-string: convert to list + set(${yaml} "${${yaml}}${indent_${level}}${member}:") + list(POP_FRONT subjson) + if(subjson STREQUAL "") + set(${yaml} "${${yaml}} []\n") + else() + set(${yaml} "${${yaml}}\n") + foreach(item ${subjson}) + set(${yaml} "${${yaml}}${indent_${level}} - ${item}\n") + endforeach() + endif() + else() + # Raw strings: save as is + set(${yaml} "${${yaml}}${indent_${level}}${member}: ${subjson}\n") + endif() else() + #---- any other data type set(${yaml} "${${yaml}}${indent_${level}}${member}: ${subjson}\n") endif() endforeach() diff --git a/cmake/yaml-filter.cmake b/cmake/yaml-filter.cmake new file mode 100644 index 00000000000000..4faeedc0da83c3 --- /dev/null +++ b/cmake/yaml-filter.cmake @@ -0,0 +1,25 @@ +# Copyright (c) 2024 Arduino SA +# SPDX-License-Identifier: Apache-2.0 + +# Simple second stage filter for YAML generation, used when generator +# expressions have been used for some of the data and the conversion to +# YAML needs to happen after cmake has completed processing. +# +# This scripts expects as input: +# - JSON_FILE: the name of the input file, in JSON format, that contains +# the expanded generator expressions. +# - YAML_FILE: the name of the final output YAML file. +# +# This script loads the Zephyr yaml module and reuses its `to_yaml()` +# function to convert the fully expanded JSON content to YAML, taking +# into account the special format that was used to store lists. + +cmake_minimum_required(VERSION 3.20.0) + +set(ZEPHYR_BASE ${CMAKE_CURRENT_LIST_DIR}/../) +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/modules") +include(yaml) + +file(READ ${JSON_FILE} json_content) +to_yaml("${json_content}" 0 yaml_out TRUE) +FILE(WRITE ${YAML_FILE} "${yaml_out}") diff --git a/doc/services/llext/build.rst b/doc/services/llext/build.rst index eae96259d245dc..1357ab85979a27 100644 --- a/doc/services/llext/build.rst +++ b/doc/services/llext/build.rst @@ -152,15 +152,34 @@ directory. It's a tarball that contains the headers and compile flags needed to build extensions. The extension developer can then include the headers and use the compile flags in their build system to build the extension. +EDK definition files +-------------------- + +The EDK includes several convenience files which define a set of variables that +contain the compile flags needed by the project, as well as other build-related +information, when enabled. The information is currently exported in the +following formats: + +- ``Makefile.cflags``, for Makefile-based projects; +- ``cmake.:cflags``, for CMake-based projects. + +Paths to the headers and flags are prefixed by the EDK root directory. This is +automatically obtained for CMake projects from ``CMAKE_CURRENT_LIST_DIR``; +other formats refer to an ``LLEXT_EDK_INSTALL_DIR`` variable, which must be set +by the user with the path where the EDK is installed before including the +generated file. + +.. note:: + The ``LLEXT_EDK`` prefix in the variable name may be changed with the + :kconfig:option:`CONFIG_LLEXT_EDK_NAME` option. + Compile flags ------------- -The EDK includes the convenience files ``cmake.cflags`` (for CMake-based -projects) and ``Makefile.cflags`` (for Make-based ones), which define a set of -variables that contain the compile flags needed by the project. The full list -of flags needed to build an extension is provided by ``LLEXT_CFLAGS``. Also -provided is a more granular set of flags that can be used in support of -different use cases, such as when building mocks for unit tests: +The full list of flags needed to build an extension is provided by +``LLEXT_CFLAGS``. Also provided is a more granular set of flags that can be +used in support of different use cases, such as when building mocks for unit +tests: ``LLEXT_INCLUDE_CFLAGS`` @@ -194,6 +213,34 @@ different use cases, such as when building mocks for unit tests: ``LLEXT_ALL_INCLUDE_CFLAGS``, ``LLEXT_GENERATED_IMACROS_CFLAGS`` and ``LLEXT_BASE_CFLAGS``. +Target and build information +---------------------------- + +The EDK includes information that identifies the target of the current Zephyr +build and the used toolchain. The following variables are currently defined, +which mirror the information available in the Zephyr build system: + +``LLEXT_EDK_BOARD_NAME`` + The board name used in the Zephyr build. + +``LLEXT_EDK_BOARD_TARGET`` + The fully qualified board target used in the Zephyr build. + +``LLEXT_EDK_TOOLCHAIN_NAME`` + The toolchain variant used in the Zephyr build. + +Depending on the toolchain, the following variables may also be exported: + +``LLEXT_EDK_TOOLCHAIN_TARGET`` + The target tuple for the compiler used in the Zephyr build. + +``LLEXT_EDK_TOOLCHAIN_TRIPLE`` + The target triple for the compiler used in the Zephyr build. + +.. note:: + The ``LLEXT_EDK`` prefix in the variable names may be changed with the + :kconfig:option:`CONFIG_LLEXT_EDK_NAME` option. + .. _llext_kconfig_edk: LLEXT EDK Kconfig options @@ -202,7 +249,8 @@ LLEXT EDK Kconfig options The LLEXT EDK can be configured using the following Kconfig options: :kconfig:option:`CONFIG_LLEXT_EDK_NAME` - The name of the generated EDK tarball. + The name of the generated EDK tarball. This is also used as the prefix for + several variables defined in the EDK files. :kconfig:option:`CONFIG_LLEXT_EDK_USERSPACE_ONLY` If set, the EDK will include headers that do not contain code to route diff --git a/scripts/schemas/build-schema.yml b/scripts/schemas/build-schema.yml index 5086afbb25b178..b07d5754454f18 100644 --- a/scripts/schemas/build-schema.yml +++ b/scripts/schemas/build-schema.yml @@ -71,6 +71,19 @@ mapping: type: seq sequence: - type: str + llext-edk: + type: map + mapping: + cflags: + type: seq + sequence: + - type: str + file: + type: str + include-dirs: + type: seq + sequence: + - type: str sysbuild: type: bool toolchain: diff --git a/tests/cmake/yaml/CMakeLists.txt b/tests/cmake/yaml/CMakeLists.txt index 04873efa9a9ec5..cacddc84b64ed3 100644 --- a/tests/cmake/yaml/CMakeLists.txt +++ b/tests/cmake/yaml/CMakeLists.txt @@ -2,24 +2,34 @@ cmake_minimum_required(VERSION 3.20.0) -find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) -project(zephyr_yaml_test) -target_sources(app PRIVATE ${ZEPHYR_BASE}/misc/empty_file.c) +if(NOT CMAKE_SCRIPT_MODE_FILE) + # Project mode initialization (main CMake invocation) + find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) + project(zephyr_yaml_test) + target_sources(app PRIVATE ${ZEPHYR_BASE}/misc/empty_file.c) + message(STATUS "Run 1 -------------\n CMake PROJECT mode\n----------------------") +else() + # Script mode initialization (re-run) + set(ZEPHYR_BASE ${CMAKE_CURRENT_LIST_DIR}/../../../) + list(APPEND CMAKE_MODULE_PATH "${ZEPHYR_BASE}/cmake/modules") + include(yaml) + message(STATUS "Run 2 ------------\n CMake SCRIPT mode\n---------------------") +endif() set_property(GLOBAL PROPERTY EXPECTED_ERROR 0) macro(message) if(DEFINED expect_failure) - if(${ARGV0} STREQUAL FATAL_ERROR) - if("${ARGV1}" STREQUAL "${expect_failure}") - # This is an expected error. - get_property(error_count GLOBAL PROPERTY EXPECTED_ERROR) - math(EXPR error_count "${error_count} + 1") - set_property(GLOBAL PROPERTY EXPECTED_ERROR ${error_count}) - return() - else() - _message("Unexpected error occurred") - endif() + if(${ARGV0} STREQUAL FATAL_ERROR) + if("${ARGV1}" STREQUAL "${expect_failure}") + # This is an expected error. + get_property(error_count GLOBAL PROPERTY EXPECTED_ERROR) + math(EXPR error_count "${error_count} + 1") + set_property(GLOBAL PROPERTY EXPECTED_ERROR ${error_count}) + return() + else() + _message("Unexpected error occurred") + endif() endif() endif() _message(${ARGN}) @@ -154,7 +164,7 @@ function(test_setting_string) ) set(new_value "A new string") - yaml_set(actual NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create + yaml_set(NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create KEY cmake test set key-string VALUE ${new_value} ) @@ -178,7 +188,7 @@ function(test_setting_list_strings) ) set(new_value "A" "new" "list" "of" "strings") - yaml_set(actual NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create + yaml_set(NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create KEY cmake test set key-list-string LIST ${new_value} ) @@ -204,13 +214,64 @@ function(test_setting_list_strings) endforeach() endfunction() +function(test_append_list_strings) + yaml_create(FILE ${CMAKE_BINARY_DIR}/${CMAKE_CURRENT_FUNCTION}_test_create.yaml + NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create + ) + + # start with entries that will be overwritten + yaml_set(NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create + KEY cmake test set key-list-string + LIST dropped entries + ) + + set(new_value "A" "new" "list" "of" "strings") + list(GET new_value 0 first) + list(SUBLIST new_value 1 -1 others) + + # re-initialize the list with the first correct value + yaml_set(NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create + KEY cmake test set key-list-string + LIST ${first} + ) + + # append the rest of the values + foreach(entry IN LISTS others) + yaml_set(NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create + KEY cmake test set key-list-string + APPEND LIST ${entry} + ) + endforeach() + + yaml_save(NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create) + + # Read-back the yaml and verify the value. + yaml_load(FILE ${CMAKE_BINARY_DIR}/${CMAKE_CURRENT_FUNCTION}_test_create.yaml + NAME ${CMAKE_CURRENT_FUNCTION}_readback + ) + + yaml_length(readback NAME ${CMAKE_CURRENT_FUNCTION}_readback KEY cmake test set key-list-string) + + test_assert(TEST 5 EQUAL ${readback} + COMMENT "readback yaml list length does not match original." + ) + + yaml_get(readback NAME ${CMAKE_CURRENT_FUNCTION}_readback KEY cmake test set key-list-string) + + foreach(a e IN ZIP_LISTS readback new_value) + test_assert(TEST "${e}" STREQUAL "${a}" + COMMENT "list values mismatch." + ) + endforeach() +endfunction() + function(test_setting_int) yaml_create(FILE ${CMAKE_BINARY_DIR}/${CMAKE_CURRENT_FUNCTION}_test_create.yaml NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create ) set(new_value 42) - yaml_set(actual NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create + yaml_set(NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create KEY cmake test set key-int VALUE ${new_value} ) @@ -234,7 +295,7 @@ function(test_setting_list_int) ) set(new_value 42 41 40 2 10) - yaml_set(actual NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create + yaml_set(NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create KEY cmake test set key-list-int LIST ${new_value} ) @@ -266,7 +327,7 @@ function(test_setting_empty_value) ) set(new_value) - yaml_set(actual NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create + yaml_set(NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create KEY cmake test set key-int VALUE ${new_value} ) @@ -320,6 +381,108 @@ function(test_setting_empty_list) endforeach() endfunction() +function(test_setting_genexes) + set(file ${CMAKE_BINARY_DIR}/test_setting_genexes.yaml) + yaml_create(FILE ${file} + NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create + ) + + set(new_value "fixed string") + yaml_set(NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create + KEY cmake test set key-string + VALUE ${new_value} + ) + + set_property(TARGET app PROPERTY expanding_str "expanded genex") + set(new_value "$") + yaml_set(NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create + KEY cmake test set key-string-genex + GENEX VALUE ${new_value} + ) + + # create the list by appending in several steps to test conversion + # from JSON list to genex stringified list + yaml_set(NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create + KEY cmake test set key-list-string-genex + LIST "A" + ) + set_property(TARGET app PROPERTY expanding_list "list" "of") + yaml_set(NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create + KEY cmake test set key-list-string-genex + APPEND GENEX LIST "new" "$" + ) + yaml_set(NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create + KEY cmake test set key-list-string-genex + APPEND LIST "strings" + ) + + yaml_save(NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create TARGET app) + + # Read-back the yaml immediately and verify the genex values are NOT present + # (genexes are expanded at generation time) + yaml_load(FILE ${file} + NAME ${CMAKE_CURRENT_FUNCTION}_readback + ) + + yaml_get(readback NAME ${CMAKE_CURRENT_FUNCTION}_readback KEY cmake test set key-string) + set(expected "fixed string") + + test_assert(TEST ${expected} STREQUAL ${readback} + COMMENT "yaml key value does not match expectation." + ) + + yaml_length(readback NAME ${CMAKE_CURRENT_FUNCTION}_readback KEY cmake test set key-list-string-genex) + set(expected cmake-test-set-key-list-string-genex-NOTFOUND) + + test_assert(TEST ${expected} STREQUAL ${readback} + COMMENT "Expected -NOTFOUND, but something was found." + ) + + yaml_get(readback NAME ${CMAKE_CURRENT_FUNCTION}_readback KEY cmake test set key-string-genex) + set(expected cmake-test-set-key-string-genex-NOTFOUND) + + test_assert(TEST ${expected} STREQUAL ${readback} + COMMENT "Expected -NOTFOUND, but something was found." + ) +endfunction() + +function(test_verify_genexes) + set(file ${CMAKE_BINARY_DIR}/test_setting_genexes.yaml) + yaml_load(FILE ${file} + NAME ${CMAKE_CURRENT_FUNCTION}_readback + ) + + yaml_get(readback NAME ${CMAKE_CURRENT_FUNCTION}_readback KEY cmake test set key-string) + set(expected "fixed string") + + test_assert(TEST ${expected} STREQUAL ${readback} + COMMENT "yaml key value does not match expectation." + ) + + set(expected "expanded genex") + yaml_get(readback NAME ${CMAKE_CURRENT_FUNCTION}_readback KEY cmake test set key-string-genex) + + test_assert(TEST ${expected} STREQUAL ${readback} + COMMENT "new yaml value does not match readback value." + ) + + set(expected "A" "new" "list" "of" "strings") + list(LENGTH expected exp_len) + + yaml_get(readback NAME ${CMAKE_CURRENT_FUNCTION}_readback KEY cmake test set key-list-string-genex) + yaml_length(act_len NAME ${CMAKE_CURRENT_FUNCTION}_readback KEY cmake test set key-list-string-genex) + + test_assert(TEST ${exp_len} EQUAL ${act_len} + COMMENT "yaml list length does not match expectation." + ) + + foreach(a e IN ZIP_LISTS readback expected) + test_assert(TEST "${e}" STREQUAL "${a}" + COMMENT "list values mismatch." + ) + endforeach() +endfunction() + function(test_set_remove_int) yaml_create(FILE ${CMAKE_BINARY_DIR}/${CMAKE_CURRENT_FUNCTION}_test_create.yaml NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create @@ -365,7 +528,7 @@ function(test_fail_missing_filename) yaml_create(NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create) set(new_value 42) - yaml_set(actual NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create + yaml_set(NAME ${CMAKE_CURRENT_FUNCTION}_yaml-create KEY cmake test set key-int VALUE ${new_value} ) @@ -398,8 +561,26 @@ test_setting_list_int() test_setting_empty_value() test_setting_empty_list() +test_append_list_strings() + test_set_remove_int() test_fail_missing_filename() -target_sources(app PRIVATE src/main.c) +# Generator expressions cannot be tested immediately, since they are expanded +# at project generation time. The script mode re-run is delayed until after +# the associated target has been built, and the verification is performed at +# that time. +if(NOT CMAKE_SCRIPT_MODE_FILE) + # create a file containing genexes + test_setting_genexes() + + # Spawn a new CMake instance to re-run the whole test suite in script mode + # and verify the genex values once the associated target is built. + add_custom_command(TARGET app POST_BUILD + COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_LIST_FILE} + ) +else() + # verify the contents of the created genex file + test_verify_genexes() +endif() diff --git a/tests/cmake/yaml/src/main.c b/tests/cmake/yaml/src/main.c deleted file mode 100644 index 0bed5a30798fcf..00000000000000 --- a/tests/cmake/yaml/src/main.c +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright (c) 2024 Nordic Semiconductor ASA - * - * SPDX-License-Identifier: Apache-2.0 - */ - -void test_main(void) -{ - /* Intentionally left empty, we are testing CMake. */ -}