Merge branch 'feature/std_filesystem_supported_v5.4' into 'release/v5.4'

storage: make std::filesystem supported (v5.4)

See merge request espressif/esp-idf!35430

components/fatfs/vfs/vfs_fat.c

@@ -39,7 +39,7 @@ typedef struct {
     FATFS fs;           /* fatfs library FS structure */
     char tmp_path_buf[FILENAME_MAX+3];  /* temporary buffer used to prepend drive name to the path */
     char tmp_path_buf2[FILENAME_MAX+3]; /* as above; used in functions which take two path arguments */
-    bool *o_append;  /* O_APPEND is stored here for each max_files entries (because O_APPEND is not compatible with FA_OPEN_APPEND) */
+    uint32_t *flags; /* file descriptor flags, array of max_files size */
 #ifdef CONFIG_VFS_SUPPORT_DIR
     char dir_path[FILENAME_MAX]; /* variable to store path of opened directory*/
     struct cached_data cached_fileinfo;
@@ -85,6 +85,7 @@ static int vfs_fat_open(void* ctx, const char * path, int flags, int mode);
 static int vfs_fat_close(void* ctx, int fd);
 static int vfs_fat_fstat(void* ctx, int fd, struct stat * st);
 static int vfs_fat_fsync(void* ctx, int fd);
+static int vfs_fat_fcntl(void* ctx, int fd, int cmd, int arg);
 #ifdef CONFIG_VFS_SUPPORT_DIR
 static int vfs_fat_stat(void* ctx, const char * path, struct stat * st);
 static int vfs_fat_link(void* ctx, const char* n1, const char* n2);
@@ -170,6 +171,7 @@ static const esp_vfs_fs_ops_t s_vfs_fat = {
     .open_p = &vfs_fat_open,
     .close_p = &vfs_fat_close,
     .fstat_p = &vfs_fat_fstat,
+    .fcntl_p = &vfs_fat_fcntl,
     .fsync_p = &vfs_fat_fsync,
 #ifdef CONFIG_VFS_SUPPORT_DIR
     .dir = &s_vfs_fat_dir,
@@ -199,19 +201,19 @@ esp_err_t esp_vfs_fat_register_cfg(const esp_vfs_fat_conf_t* conf, FATFS** out_f
         return ESP_ERR_NO_MEM;
     }
     memset(fat_ctx, 0, ctx_size);
-    fat_ctx->o_append = ff_memalloc(max_files * sizeof(bool));
-    if (fat_ctx->o_append == NULL) {
+    fat_ctx->flags = ff_memalloc(max_files * sizeof(*fat_ctx->flags));
+    if (fat_ctx->flags == NULL) {
         free(fat_ctx);
         return ESP_ERR_NO_MEM;
     }
-    memset(fat_ctx->o_append, 0, max_files * sizeof(bool));
+    memset(fat_ctx->flags, 0, max_files * sizeof(*fat_ctx->flags));
     fat_ctx->max_files = max_files;
     strlcpy(fat_ctx->fat_drive, conf->fat_drive, sizeof(fat_ctx->fat_drive) - 1);
     strlcpy(fat_ctx->base_path, conf->base_path, sizeof(fat_ctx->base_path) - 1);
 
     esp_err_t err = esp_vfs_register_fs(conf->base_path, &s_vfs_fat, ESP_VFS_FLAG_CONTEXT_PTR | ESP_VFS_FLAG_STATIC, fat_ctx);
     if (err != ESP_OK) {
-        free(fat_ctx->o_append);
+        free(fat_ctx->flags);
         free(fat_ctx);
         return err;
     }
@@ -239,7 +241,7 @@ esp_err_t esp_vfs_fat_unregister_path(const char* base_path)
         return err;
     }
     _lock_close(&fat_ctx->lock);
-    free(fat_ctx->o_append);
+    free(fat_ctx->flags);
     free(fat_ctx);
     s_fat_ctxs[ctx] = NULL;
     return ESP_OK;
@@ -427,7 +429,7 @@ static int vfs_fat_open(void* ctx, const char * path, int flags, int mode)
     // Other VFS drivers handles O_APPEND well (to the best of my knowledge),
     // therefore this flag is stored here (at this VFS level) in order to save
     // memory.
-    fat_ctx->o_append[fd] = (flags & O_APPEND) == O_APPEND;
+    fat_ctx->flags[fd] = (flags & (O_APPEND | O_ACCMODE));
     _lock_release(&fat_ctx->lock);
     return fd;
 }
@@ -438,7 +440,7 @@ static ssize_t vfs_fat_write(void* ctx, int fd, const void * data, size_t size)
     FIL* file = &fat_ctx->files[fd];
     FRESULT res;
     _lock_acquire(&fat_ctx->lock);
-    if (fat_ctx->o_append[fd]) {
+    if (fat_ctx->flags[fd] & O_APPEND) {
         if ((res = f_lseek(file, f_size(file))) != FR_OK) {
             ESP_LOGD(TAG, "%s: fresult=%d", __func__, res);
             errno = fresult_to_errno(res);
@@ -672,6 +674,26 @@ static int vfs_fat_fstat(void* ctx, int fd, struct stat * st)
     return 0;
 }
 
+static int vfs_fat_fcntl(void* ctx, int fd, int cmd, int arg)
+{
+    vfs_fat_ctx_t* fat_ctx = (vfs_fat_ctx_t*) ctx;
+    switch (cmd) {
+        case F_GETFL:
+            return fat_ctx->flags[fd];
+        case F_SETFL:
+            fat_ctx->flags[fd] = arg;
+            return 0;
+        // no-ops:
+        case F_SETLK:
+        case F_SETLKW:
+        case F_GETLK:
+            return 0;
+        default:
+            errno = EINVAL;
+            return -1;
+    }
+}
+
 #ifdef CONFIG_VFS_SUPPORT_DIR
 
 static inline mode_t get_stat_mode(bool is_dir)

components/newlib/realpath.c

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -9,6 +9,7 @@
 #include <string.h>
 #include <stdlib.h>
 #include <assert.h>
+#include <dirent.h>
 #include <sys/param.h>
 
 /* realpath logic:
@@ -122,3 +123,20 @@ int chdir(const char *path)
     errno = ENOSYS;
     return -1;
 }
+
+/* std::filesystem functions call chmod and exit with an exception if it fails,
+ * so not failing with ENOSYS seems a better solution.
+ */
+int chmod(const char *path, mode_t mode)
+{
+    return 0;
+}
+
+/* As a workaround for libstdc++ being built with _GLIBCXX_HAVE_DIRFD,
+ * we have to provide at least a stub for dirfd function.
+ */
+int dirfd(DIR *dirp)
+{
+    errno = ENOSYS;
+    return -1;
+}

components/newlib/sysconf.c

@@ -1,13 +1,18 @@
 /*
- * SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
 
 #include <unistd.h>
 #include <errno.h>
+#include <limits.h>
+#include "esp_err.h"
+#include "esp_log.h"
 #include "sdkconfig.h"
 
+static const char *TAG = "sysconf";
+
 #ifdef CONFIG_FREERTOS_UNICORE
 #define CPU_NUM 1
 #else
@@ -25,3 +30,24 @@ long sysconf(int arg)
         return -1;
     }
 }
+
+// pathconf
+long fpathconf(int fildes, int name)
+{
+    if (name == _PC_PATH_MAX) {
+        return PATH_MAX;
+    }
+    ESP_LOGW(TAG, "fpathconf: unsupported name %d", name);
+    errno = EINVAL;
+    return -1;
+}
+
+long pathconf(const char *path, int name)
+{
+    if (name == _PC_PATH_MAX) {
+        return PATH_MAX;
+    }
+    ESP_LOGW(TAG, "pathconf: unsupported name %d", name);
+    errno = EINVAL;
+    return -1;
+}

docs/en/api-guides/cplusplus.rst

@@ -13,6 +13,7 @@ The following C++ features are supported:
 - :ref:`cplusplus_multithreading`
 - :ref:`cplusplus_rtti`
 - :doc:`thread-local-storage` (``thread_local`` keyword)
+- :ref:`cplusplus_filesystem`
 - All C++ features implemented by GCC, except for some :ref:`cplusplus_limitations`. See `GCC documentation <https://gcc.gnu.org/projects/cxx-status.html>`_ for details on features implemented by GCC.
 
 
@@ -94,6 +95,21 @@ Enabling this option compiles all C++ files with RTTI support enabled, which all
 
 See :example:`cxx/rtti` for an example of using RTTI in ESP-IDF. Specifically, this example demonstrates how to use the RTTI feature in ESP-IDF, enabling compile time support for RTTI, and showing how to print demangled type names of objects and functions, and how dynamic_cast behaves with objects of two classes derived from a common base class.
 
+.. _cplusplus_filesystem:
+
+Filesystem Library
+------------------
+
+C++ Filesystem library (``#include <filesystem>``) features are supported in ESP-IDF, with the following exceptions:
+
+- Since symbolic and hard links are not supported in ESP-IDF, related functions are not implemented.
+- ``std::filesystem::space`` is not implemented.
+- ``std::filesystem::resize_file`` is not implemented.
+- ``std::filesystem::current_path`` always returns ``/``. Setting the current path is not supported.
+- ``std::filesystem::permissions`` doesn't support setting file permissions.
+
+Note that the choice of the filesystem also affects the behavior of the filesystem library. For example, SPIFFS filesystem has limited support for directories, therefore the related std::filesystem functions may not work as they do on a filesystem which does support directories.
+
 Developing in C++
 -----------------
 
@@ -186,7 +202,6 @@ Limitations
 - Linker script generator does not support function level placements for functions with C++ linkage.
 - Various section attributes (such as ``IRAM_ATTR``) are ignored when used with template functions.
 - Vtables are placed into Flash and are not accessible when the flash cache is disabled. Therefore, virtual function calls should be avoided in :ref:`iram-safe-interrupt-handlers`. Placement of Vtables cannot be adjusted using the linker script generator, yet.
-- C++ filesystem (``std::filesystem``) features are not supported.
 
 
 What to Avoid

docs/zh_CN/api-guides/cplusplus.rst

@@ -13,6 +13,7 @@ ESP-IDF 支持以下 C++ 功能：
 - :ref:`cplusplus_multithreading`
 - :ref:`cplusplus_rtti`
 - :doc:`thread-local-storage` （``thread_local`` 关键字）
+- :ref:`cplusplus_filesystem`
 - 除部分 :ref:`cplusplus_limitations`，所有由 GCC 实现的 C++ 功能均受支持。有关由 GCC 所实现功能的详细信息，请参阅 `GCC 文档 <https://gcc.gnu.org/projects/cxx-status.html>`_。
 
 
@@ -94,6 +95,21 @@ ESP-IDF 默认禁用对 RTTI 的支持，可以用 :ref:`CONFIG_COMPILER_CXX_RTT
 
 有关在 ESP-IDF 中使用 RTTI 的示例，请参阅 :example:`cxx/rtti`。该示例演示了如何在 ESP-IDF 中使用 RTTI 功能，启用编译时对 RTTI 的支持，并展示了如何打印对象和函数的去混淆类型名称，以及 dynamic_cast 在两个继承自同一基类的对象上如何表现。
 
+.. _cplusplus_filesystem:
+
+文件系统库
+----------
+
+ESP-IDF 支持 C++ 文件系统库 (``#include <filesystem>``)，但有部分功能尚未实现：
+
+- 由于 ESP-IDF 不支持符号链接和硬链接，因此相关函数未实现。
+- 未实现 ``std::filesystem::space``。
+- 未实现 ``std::filesystem::resize_file``。
+- ``std::filesystem::current_path`` 只返回 ``/``。不支持设置当前路径。
+- ``std::filesystem::permissions`` 不支持设置文件权限。
+
+请注意，文件系统的选择也会影响文件系统库的行为。例如，SPIFFS 文件系统对目录的支持有限，因此相关的 std::filesystem 函数可能无法像在支持目录的文件系统上那样正常工作。
+
 在 C++ 中进行开发
 -----------------
 
@@ -186,7 +202,6 @@ ESP-IDF 支持 ``iostream`` 功能，但应注意：
 - 链接脚本生成器不支持将具有 C++ 链接的函数单独放置在内存的特定位置。
 - 当与模板函数一起使用时，会忽略各种节属性（例如 ``IRAM_ATTR``）。
 - vtable 位于 flash 中，在禁用 flash 缓存时无法访问。因此，在 :ref:`iram-safe-interrupt-handlers` 中应避免调用虚拟函数。目前尚无法使用链接器脚本生成器调整 vtable 的放置位置。
-- 不支持 C++ 文件系统 (``std::filesystem``) 功能。
 
 
 注意事项

tools/test_apps/storage/.build-test-rules.yml

@@ -38,3 +38,15 @@ tools/test_apps/storage/sdmmc_console:
     - sdmmc
     - esp_driver_sdmmc
     - esp_driver_sdspi
+
+tools/test_apps/storage/std_filesystem:
+  enable:
+    - if: IDF_TARGET in ["esp32", "esp32c3"]
+      reason: one Xtensa and one RISC-V chip should be enough
+  disable:
+    - if: IDF_TOOLCHAIN == "clang"
+      reason: Issue with C++ exceptions on Xtensa, issue with getrandom linking on RISC-V
+  depends_components:
+    - vfs
+    - newlib
+    - fatfs

tools/test_apps/storage/std_filesystem/CMakeLists.txt

@@ -0,0 +1,8 @@
+# The following five lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.16)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+set(COMPONENTS main)
+list(PREPEND SDKCONFIG_DEFAULTS "$ENV{IDF_PATH}/tools/test_apps/configs/sdkconfig.debug_helpers" "sdkconfig.defaults")
+project(std_filesystem_test)

tools/test_apps/storage/std_filesystem/README.md

@@ -0,0 +1,70 @@
+| Supported Targets | ESP32 | ESP32-C3 |
+| ----------------- | ----- | -------- |
+
+This is a test app which verifies that std::filesystem features work in ESP-IDF. The tests are written using [Catch2](https://github.com/catchorg/Catch2) managed [component](https://components.espressif.com/components/espressif/catch2/).
+
+To run the tests:
+
+```shell
+idf.py flash monitor
+```
+
+Or, in QEMU:
+
+```shell
+idf.py qemu monitor
+```
+
+Or, using pytest:
+
+```shell
+idf.py build
+pytest --embedded-services idf,qemu --target esp32 --ignore managed_components
+```
+
+## Feature Support
+
+Please update `_cplusplus_filesystem` section in cplusplus.rst when modifying this table.
+
+| Feature                      | Supported | Tested | Comment                                                                                                       |
+|------------------------------|-----------|--------|---------------------------------------------------------------------------------------------------------------|
+| absolute                     | y         | y      |                                                                                                               |
+| canonical                    | y         | y      |                                                                                                               |
+| weakly_canonical             | y         | y      |                                                                                                               |
+| relative                     | y         | y      |                                                                                                               |
+| proximate                    | y         | y      |                                                                                                               |
+| copy                         | y         | y      | this function has complex behavior, not sure about test coverage                                              |
+| copy_file                    | y         | y      |                                                                                                               |
+| copy_symlink                 | n         | n      | symlinks are not supported                                                                                    |
+| create_directory             | y         | y      |                                                                                                               |
+| create_directories           | y         | y      |                                                                                                               |
+| create_hard_link             | n         | n      | hard links are not supported                                                                                  |
+| create_symlink               | n         | n      | symlinks are not supported                                                                                    |
+| create_directory_symlink     | n         | n      | symlinks are not supported                                                                                    |
+| current_path                 | partial   | y      | setting path is not supported in IDF                                                                          |
+| exists                       | y         | y      |                                                                                                               |
+| equivalent                   | y         | y      |                                                                                                               |
+| file_size                    | y         | y      |                                                                                                               |
+| hard_link_count              | n         | n      | hard links are not supported                                                                                  |
+| last_write_time              | y         | y      |                                                                                                               |
+| permissions                  | partial   | y      | setting permissions is not supported                                                                          |
+| read_symlink                 | n         | n      | symlinks are not supported                                                                                    |
+| remove                       | y         | y      |                                                                                                               |
+| remove_all                   | y         | y      |                                                                                                               |
+| rename                       | y         | y      |                                                                                                               |
+| resize_file                  | n         | y      | doesn't work, toolchain has to be built with _GLIBCXX_HAVE_TRUNCATE                                           |
+| space                        | n         | y      | doesn't work, toolchain has to be built with _GLIBCXX_HAVE_SYS_STATVFS_H and statvfs function must be defined |
+| status                       | y         | y      |                                                                                                               |
+| symlink_status               | n         | n      | symlinks are not supported                                                                                    |
+| temp_directory_path          | y         | y      | works if /tmp directory has been mounted                                                                      |
+| directory_iterator           | y         | y      |                                                                                                               |
+| recursive_directory_iterator | y         | y      |                                                                                                               |
+| is_block_file                | y         | y      |                                                                                                               |
+| is_character_file            | y         | y      |                                                                                                               |
+| is_directory                 | y         | y      |                                                                                                               |
+| is_empty                     | y         | y      |                                                                                                               |
+| is_fifo                      | y         | y      |                                                                                                               |
+| is_other                     | n         | n      |                                                                                                               |
+| is_regular_file              | y         | y      |                                                                                                               |
+| is_socket                    | y         | y      |                                                                                                               |
+| is_symlink                   | y         | y      |                                                                                                               |

tools/test_apps/storage/std_filesystem/main/CMakeLists.txt

@@ -0,0 +1,10 @@
+idf_component_register(SRCS
+        "test_std_filesystem_main.cpp"
+        "test_ops.cpp"
+        "test_paths.cpp"
+        "test_status.cpp"
+    INCLUDE_DIRS "."
+    PRIV_REQUIRES vfs fatfs
+    WHOLE_ARCHIVE)
+
+fatfs_create_spiflash_image(storage ${CMAKE_CURRENT_LIST_DIR}/test_fs_image FLASH_IN_PROJECT)

tools/test_apps/storage/std_filesystem/main/idf_component.yml

@@ -0,0 +1,2 @@
+dependencies:
+  espressif/catch2: "^3.7.0"

tools/test_apps/storage/std_filesystem/main/test_fs_image/file

@@ -0,0 +1 @@
+1234567890