Merge branch 'feat/fatfs_support_sector_less_than_128' into 'master'

feat(fatfs): enable partition handling for sectors less than 128

Closes IDF-8538

See merge request espressif/esp-idf!33569

components/fatfs/fatfsgen.py

@@ -1,21 +1,28 @@
 #!/usr/bin/env python
 # SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 import os
 from datetime import datetime
-from typing import Any, List, Optional
+from typing import Any
+from typing import List
+from typing import Optional
 
 from fatfs_utils.boot_sector import BootSector
 from fatfs_utils.exceptions import NoFreeClusterException
 from fatfs_utils.fat import FAT
 from fatfs_utils.fatfs_state import FATFSState
 from fatfs_utils.fs_object import Directory
 from fatfs_utils.long_filename_utils import get_required_lfn_entries_count
-from fatfs_utils.utils import (BYTES_PER_DIRECTORY_ENTRY, FATFS_INCEPTION, FATFS_MIN_ALLOC_UNIT,
-                               RESERVED_CLUSTERS_COUNT, FATDefaults, get_args_for_partition_generator,
-                               get_fat_sectors_count, get_non_data_sectors_cnt, read_filesystem,
-                               required_clusters_count)
+from fatfs_utils.utils import BYTES_PER_DIRECTORY_ENTRY
+from fatfs_utils.utils import FATDefaults
+from fatfs_utils.utils import FATFS_INCEPTION
+from fatfs_utils.utils import FATFS_MIN_ALLOC_UNIT
+from fatfs_utils.utils import get_args_for_partition_generator
+from fatfs_utils.utils import get_fat_sectors_count
+from fatfs_utils.utils import get_non_data_sectors_cnt
+from fatfs_utils.utils import read_filesystem
+from fatfs_utils.utils import required_clusters_count
+from fatfs_utils.utils import RESERVED_CLUSTERS_COUNT
 
 
 def duplicate_fat_decorator(func):  # type: ignore
@@ -48,14 +55,15 @@ def __init__(self,
                  volume_label: str = FATDefaults.VOLUME_LABEL,
                  file_sys_type: str = FATDefaults.FILE_SYS_TYPE,
                  root_entry_count: int = FATDefaults.ROOT_ENTRIES_COUNT,
-                 explicit_fat_type: int = None,
+                 explicit_fat_type: Optional[int] = None,
                  media_type: int = FATDefaults.MEDIA_TYPE) -> None:
         # root directory bytes should be aligned by sector size
-        assert (root_entry_count * BYTES_PER_DIRECTORY_ENTRY) % sector_size == 0
+        assert (int(root_entry_count) * BYTES_PER_DIRECTORY_ENTRY) % sector_size == 0
         # number of bytes in the root dir must be even multiple of BPB_BytsPerSec
-        assert ((root_entry_count * BYTES_PER_DIRECTORY_ENTRY) // sector_size) % 2 == 0
+        if (int(root_entry_count) > 128):
+            assert ((int(root_entry_count) * BYTES_PER_DIRECTORY_ENTRY) // sector_size) % 2 == 0
 
-        root_dir_sectors_cnt: int = (root_entry_count * BYTES_PER_DIRECTORY_ENTRY) // sector_size
+        root_dir_sectors_cnt: int = (int(root_entry_count) * BYTES_PER_DIRECTORY_ENTRY) // sector_size
 
         self.state: FATFSState = FATFSState(sector_size=sector_size,
                                             explicit_fat_type=explicit_fat_type,

components/fatfs/host_test/main/test_fatfs.cpp

@@ -44,7 +44,10 @@ TEST_CASE("Create volume, open file, write and read back data", "[fatfs]")
 
     fr_result = f_fdisk(pdrv, part_list, work_area);
     REQUIRE(fr_result == FR_OK);
-    const MKFS_PARM opt = {(BYTE)FM_ANY, 0, 0, 0, 0};
+
+    // For host tests, include FM_SFD flag when formatting partitions smaller than 128KB.
+    // if n_root field of MKFS_PARM is set to 128 => 1 root directory sec and if set to 0(default 512) => 4 root directory sectors.
+    const MKFS_PARM opt = {(BYTE)(FM_ANY | FM_SFD), 0, 0, 128, 0};
     fr_result = f_mkfs("", &opt, work_area, sizeof(work_area)); // Use default volume
 
     // Mount the volume
@@ -56,7 +59,7 @@ TEST_CASE("Create volume, open file, write and read back data", "[fatfs]")
     REQUIRE(fr_result == FR_OK);
 
     // Generate data
-    uint32_t data_size = 100000;
+    uint32_t data_size = 1000;
 
     char *data = (char*) malloc(data_size);
     char *read = (char*) malloc(data_size);
@@ -130,7 +133,7 @@ static void prepare_fatfs(const char* partition_label, const esp_partition_t** p
 
     fr_result = f_fdisk(_pdrv, part_list, work_area);
     REQUIRE(fr_result == FR_OK);
-    const MKFS_PARM opt = {(BYTE)FM_ANY, 0, 0, 0, 0};
+    const MKFS_PARM opt = {(BYTE)(FM_ANY | FM_SFD), 0, 0, 128, 0};
     fr_result = f_mkfs(drv, &opt, work_area, sizeof(work_area)); // Use default volume
     REQUIRE(fr_result == FR_OK);
 }
@@ -222,7 +225,7 @@ TEST_CASE("Test mounting 2 volumes, writing data and formatting the 2nd one, rea
     const size_t workbuf_size = 4096;
     void *workbuf = ff_memalloc(workbuf_size);
     REQUIRE(workbuf != NULL);
-    const MKFS_PARM opt = {(BYTE)(FM_ANY | FM_SFD), 0, 0, 0, CONFIG_WL_SECTOR_SIZE};
+    const MKFS_PARM opt = {(BYTE)(FM_ANY | FM_SFD), 0, 0, 128, CONFIG_WL_SECTOR_SIZE};
     fr_result = f_mkfs(drv1, &opt, workbuf, workbuf_size);
     free(workbuf);
     workbuf = NULL;

components/fatfs/host_test/partition_table.csv

@@ -3,5 +3,5 @@
 nvs,      data, nvs,     0x9000,  0x6000,
 phy_init, data, phy,     0xf000,  0x1000,
 factory,  app,  factory, 0x10000, 1M,
-storage,  data, fat,     ,        1M,
-storage2, data, fat,     ,        1M,
+storage,  data, fat,     ,        32k,
+storage2, data, fat,     ,        32k,

components/fatfs/src/ff.c

@@ -34,7 +34,6 @@
 #error Wrong include file (ff.h).
 #endif
 
-
 /* Limits and boundaries */
 #define MAX_DIR		0x200000		/* Max size of FAT directory */
 #define MAX_DIR_EX	0x10000000		/* Max size of exFAT directory */
@@ -43,6 +42,10 @@
 #define MAX_FAT32	0x0FFFFFF5		/* Max FAT32 clusters (not specified, practical limit) */
 #define MAX_EXFAT	0x7FFFFFFD		/* Max exFAT clusters (differs from specs, implementation limit) */
 
+#define MIN_FAT12_SEC_VOL         4  /* Min size of the FAT sector volume
+                                        1 FAT, 1 root dir, 1 reserved, 1 data sector */
+#define MIN_FAT12_DATA_SEC        1  /* Min FAT data sectors */
+
 
 /* Character code support macros */
 #define IsUpper(c)		((c) >= 'A' && (c) <= 'Z')
@@ -3318,7 +3321,7 @@ static UINT check_fs (	/* 0:FAT/FAT32 VBR, 1:exFAT VBR, 2:Not FAT and valid BS,
 			&& ld_word(fs->win + BPB_RsvdSecCnt) != 0	/* Properness of reserved sectors (MNBZ) */
 			&& (UINT)fs->win[BPB_NumFATs] - 1 <= 1		/* Properness of FATs (1 or 2) */
 			&& ld_word(fs->win + BPB_RootEntCnt) != 0	/* Properness of root dir entries (MNBZ) */
-			&& (ld_word(fs->win + BPB_TotSec16) >= 128 || ld_dword(fs->win + BPB_TotSec32) >= 0x10000)	/* Properness of volume sectors (>=128) */
+			&& (ld_word(fs->win + BPB_TotSec16) >= MIN_FAT12_SEC_VOL || ld_dword(fs->win + BPB_TotSec32) >= 0x10000)	/* Properness of volume sectors (>=MIN_FAT12_SEC_VOL) */
 			&& ld_word(fs->win + BPB_FATSz16) != 0) {	/* Properness of FAT size (MNBZ) */
 				return 0;	/* It can be presumed an FAT VBR */
 		}
@@ -6034,7 +6037,11 @@ FRESULT f_mkfs (
 			}
 		}
 	}
-	if (sz_vol < 128) LEAVE_MKFS(FR_MKFS_ABORTED);	/* Check if volume size is >=128s */
+	if (n_fat == 1) {
+			if (sz_vol < MIN_FAT12_SEC_VOL) LEAVE_MKFS(FR_MKFS_ABORTED); /* Check if volume size is >= MIN_FAT12_SEC_VOLs */
+	} else {
+			if (sz_vol < (MIN_FAT12_SEC_VOL + 1)) LEAVE_MKFS(FR_MKFS_ABORTED); /* Check if volume size is >= (MIN_FAT12_SEC_VOL+1)s */
+	}
 
 	/* Now start to create an FAT volume at b_vol and sz_vol */
 
@@ -6265,7 +6272,7 @@ FRESULT f_mkfs (
 			}
 
 			/* Determine number of clusters and final check of validity of the FAT sub-type */
-			if (sz_vol < b_data + pau * 16 - b_vol) LEAVE_MKFS(FR_MKFS_ABORTED);	/* Too small volume? */
+			if (sz_vol < b_data + pau * MIN_FAT12_DATA_SEC - b_vol) LEAVE_MKFS(FR_MKFS_ABORTED);	/* Too small volume? */
 			n_clst = ((DWORD)sz_vol - sz_rsv - sz_fat * n_fat - sz_dir) / pau;
 			if (fsty == FS_FAT32) {
 				if (n_clst <= MAX_FAT16) {	/* Too few clusters for FAT32? */

components/fatfs/test_apps/flash_wl/main/CMakeLists.txt

@@ -1,4 +1,4 @@
-idf_component_register(SRCS "test_fatfs_flash_wl.c"
+idf_component_register(SRCS "test_fatfs_flash_wl.c" "test_fatfs_small_partition.c"
                        INCLUDE_DIRS "."
                        PRIV_REQUIRES unity spi_flash fatfs vfs test_fatfs_common
                        WHOLE_ARCHIVE)

components/fatfs/test_apps/flash_wl/main/test_fatfs_small_partition.c

@@ -0,0 +1,76 @@
+/*
+ * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <sys/unistd.h>
+#include "unity.h"
+#include "esp_vfs_fat.h"
+
+static wl_handle_t s_test_wl_handle;
+static void test_setup(void)
+{
+  // With this configuration, for 32k partition size,
+  // 4 sectors will be used for WL and 4 sectors for FATFS
+  // (1 FAT,  1 root directory, 1 reserved and 1 data sector)
+    esp_vfs_fat_mount_config_t mount_config = {
+        .format_if_mount_failed = true,
+        .max_files = 5,
+        .use_one_fat = true,
+    };
+
+    TEST_ESP_OK(esp_vfs_fat_spiflash_format_cfg_rw_wl("/spiflash", "storage1", &mount_config));
+    TEST_ESP_OK(esp_vfs_fat_spiflash_mount_rw_wl("/spiflash", "storage1", &mount_config, &s_test_wl_handle));
+}
+
+static void test_teardown(void)
+{
+    TEST_ESP_OK(esp_vfs_fat_spiflash_unmount_rw_wl("/spiflash", s_test_wl_handle));
+}
+
+static void test_write_data_sec(int num_data_sec)
+{
+    int fd = open("/spiflash/test.txt", O_CREAT | O_WRONLY);
+    TEST_ASSERT_NOT_EQUAL(-1, fd);
+
+    // Generate data
+    uint32_t data_size = 4096*num_data_sec;
+
+    char *data = (char*) malloc(data_size);
+    char *read_data = (char*) malloc(data_size);
+
+    for(uint32_t i = 0; i < (data_size); i += sizeof(i))
+    {
+        *((uint32_t*)(data + i)) = i;
+    }
+    ssize_t wr = write(fd, data, data_size);
+    if (num_data_sec == 1) {
+        TEST_ASSERT_EQUAL(data_size, wr);
+    } else {
+        TEST_ASSERT_NOT_EQUAL(data_size, wr);
+    }
+    TEST_ASSERT_EQUAL(0, close(fd));
+
+    fd = open("/spiflash/test.txt", O_RDONLY);
+    int r = read(fd, read_data, data_size);
+    if (num_data_sec == 1) {
+        TEST_ASSERT_EQUAL(data_size, r);
+    } else {
+        TEST_ASSERT_NOT_EQUAL(data_size, r);
+    }
+    TEST_ASSERT_EQUAL(0, strcmp(data, read_data));
+    TEST_ASSERT_EQUAL(0, close(fd));
+}
+
+TEST_CASE("(WL) can format small partition and read-write data", "[fatfs][wear_levelling][timeout=120]")
+{
+    test_setup();
+    test_write_data_sec(1); //for 1 data sectors, write and read func should work
+    test_write_data_sec(2); //for 2 data sectors, write and read func should fail
+    test_teardown();
+}

components/fatfs/test_apps/flash_wl/partitions.csv

@@ -2,3 +2,4 @@
 factory,  app,  factory, 0x10000, 768k,
 storage,  data, fat,     ,        528k,
 storage2, data, fat,     ,        528k,
+storage1, data, fat,     ,        32k,

components/fatfs/vfs/vfs_fat_spiflash.c

@@ -15,6 +15,10 @@
 #include "wear_levelling.h"
 #include "diskio_wl.h"
 
+// If the available sectors based on partition size are less than 128,
+// the root directory sector should be set to 1.
+#define MIN_REQ_SEC    128
+
 static const char* TAG = "vfs_fat_spiflash";
 
 static vfs_fat_spiflash_ctx_t *s_ctx[FF_VOLUMES] = {};
@@ -74,7 +78,7 @@ vfs_fat_spiflash_ctx_t* get_vfs_fat_spiflash_ctx(wl_handle_t wlhandle)
     return NULL;
 }
 
-static esp_err_t s_f_mount_rw(FATFS *fs, const char *drv, const esp_vfs_fat_mount_config_t *mount_config, vfs_fat_x_ctx_flags_t *out_flags)
+static esp_err_t s_f_mount_rw(FATFS *fs, const char *drv, const esp_vfs_fat_mount_config_t *mount_config, vfs_fat_x_ctx_flags_t *out_flags, size_t sec_num)
 {
     FRESULT fresult = f_mount(fs, drv, 1);
     if (fresult != FR_OK) {
@@ -93,7 +97,13 @@ static esp_err_t s_f_mount_rw(FATFS *fs, const char *drv, const esp_vfs_fat_moun
 
         size_t alloc_unit_size = esp_vfs_fat_get_allocation_unit_size(CONFIG_WL_SECTOR_SIZE, mount_config->allocation_unit_size);
         ESP_LOGI(TAG, "Formatting FATFS partition, allocation unit size=%d", alloc_unit_size);
-        const MKFS_PARM opt = {(BYTE)(FM_ANY | FM_SFD), (mount_config->use_one_fat ? 1 : 2), 0, 0, alloc_unit_size};
+        UINT root_dir_entries;
+        if (CONFIG_WL_SECTOR_SIZE == 512) {
+            root_dir_entries = 16;
+        } else {
+            root_dir_entries = 128;
+        }
+        const MKFS_PARM opt = {(BYTE)(FM_ANY | FM_SFD), (mount_config->use_one_fat ? 1 : 2), 0, (sec_num <= MIN_REQ_SEC ? root_dir_entries : 0), alloc_unit_size};
         fresult = f_mkfs(drv, &opt, workbuf, workbuf_size);
         free(workbuf);
         workbuf = NULL;
@@ -157,8 +167,9 @@ esp_err_t esp_vfs_fat_spiflash_mount_rw_wl(const char* base_path,
 
     vfs_fat_x_ctx_flags_t flags = 0;
 
+    size_t sec_num = wl_size(*wl_handle) / wl_sector_size(*wl_handle);
     // Try to mount partition
-    ret = s_f_mount_rw(fs, drv, mount_config, &flags);
+    ret = s_f_mount_rw(fs, drv, mount_config, &flags, sec_num);
     if (ret != ESP_OK) {
         goto fail;
     }
@@ -224,6 +235,7 @@ esp_err_t esp_vfs_fat_spiflash_format_cfg_rw_wl(const char* base_path, const cha
 
     wl_handle_t temp_handle = WL_INVALID_HANDLE;
     uint32_t id = FF_VOLUMES;
+    size_t sec_num = 0;
 
     bool found = s_get_context_id_by_label(partition_label, &id);
     if (!found) {
@@ -239,6 +251,7 @@ esp_err_t esp_vfs_fat_spiflash_format_cfg_rw_wl(const char* base_path, const cha
         }
         ESP_RETURN_ON_ERROR(esp_vfs_fat_spiflash_mount_rw_wl(base_path, partition_label, mount_cfg, &temp_handle), TAG, "Failed to mount");
         found = s_get_context_id_by_label(partition_label, &id);
+        sec_num = wl_size(temp_handle) / wl_sector_size(temp_handle);
         assert(found);
         if (s_ctx[id]->flags & FORMATTED_DURING_LAST_MOUNT) {
             ESP_LOGD(TAG, "partition was formatted during mounting, skipping another format");
@@ -250,6 +263,8 @@ esp_err_t esp_vfs_fat_spiflash_format_cfg_rw_wl(const char* base_path, const cha
         if (cfg) {
             s_ctx[id]->mount_config = *cfg;
         }
+        temp_handle = s_ctx[id]->wlhandle;
+        sec_num = wl_size(temp_handle) / wl_sector_size(temp_handle);
     }
 
     //unmount
@@ -266,15 +281,21 @@ esp_err_t esp_vfs_fat_spiflash_format_cfg_rw_wl(const char* base_path, const cha
     }
     size_t alloc_unit_size = esp_vfs_fat_get_allocation_unit_size(CONFIG_WL_SECTOR_SIZE, s_ctx[id]->mount_config.allocation_unit_size);
     ESP_LOGI(TAG, "Formatting FATFS partition, allocation unit size=%d", alloc_unit_size);
-    const MKFS_PARM opt = {(BYTE)(FM_ANY | FM_SFD), (s_ctx[id]->mount_config.use_one_fat ? 1 : 2), 0, 0, alloc_unit_size};
+    UINT root_dir_entries;
+    if (CONFIG_WL_SECTOR_SIZE == 512) {
+        root_dir_entries = 16;
+    } else {
+        root_dir_entries = 128;
+    }
+    const MKFS_PARM opt = {(BYTE)(FM_ANY | FM_SFD), (s_ctx[id]->mount_config.use_one_fat ? 1 : 2), 0, (sec_num <= MIN_REQ_SEC ? root_dir_entries : 0), alloc_unit_size};
     fresult = f_mkfs(drv, &opt, workbuf, workbuf_size);
     free(workbuf);
     workbuf = NULL;
     ESP_GOTO_ON_FALSE(fresult == FR_OK, ESP_FAIL, mount_back, TAG, "f_mkfs failed (%d)", fresult);
 
 mount_back:
     if (partition_was_mounted) {
-        esp_err_t err = s_f_mount_rw(s_ctx[id]->fs, drv, &s_ctx[id]->mount_config, NULL);
+        esp_err_t err = s_f_mount_rw(s_ctx[id]->fs, drv, &s_ctx[id]->mount_config, NULL, sec_num);
         if (err != ESP_OK) {
             ESP_LOGE(TAG, "failed to mount back, go to recycle");
             goto recycle;

docs/en/api-guides/file-system-considerations.rst

@@ -50,7 +50,7 @@ The most significant properties and features of above-mentioned file systems are
       - Integrated
       - Integrated
     * - Minimum partition size
-      - * 128 sectors With wear levelling on (WL sector=4096B):
+      - * 8 sectors with wear levelling on (4 FATFS sectors + 4 WL sectors with WL sector size = 4096B)
         * plus 4 sectors at least
         * real number given by WL configuration (Safe, Perf)
       - * 6 logical blocks

docs/en/api-reference/storage/fatfs.rst

@@ -157,6 +157,32 @@ Usage::
 
 Parameter --verbose prints detailed information from boot sector of the FatFs image to the terminal before folder structure is generated.
 
+FATFS Minimum Partition Size and Limits
+---------------------------------------
+
+The FATFS component supports FAT12, FAT16, and FAT32 file system types. The file system type is determined by the number of clusters (calculated as data sectors divided by sectors per cluster) on the volume. The minimum partition size is defined by the number of sectors allocated to FAT tables, root directories and data clusters.
+
+* The minimum supported size for a FAT partition with wear leveling enabled is 32 KB for a sector size of 4096 bytes. For a sector size of 512 bytes, the minimum partition size varies based on the WL configuration: 20 KB for Performance mode and 28 KB for Safety mode (requiring 2 extra sectors).
+* For a partition with wear leveling enabled, 4 sectors will be reserved for wear-leveling operations, and 4 sectors will be used by the FATFS (1 reserved sector, 1 FAT sector, 1 root directory sector and 1 data sector).
+* Increasing the partition size will allocate additional data sectors, allowing for more storage space.
+* For partition sizes less than 528 KB, 1 root directory sector will be allocated; for larger partitions, 4 root directory sectors will be used.
+* By default, two FAT sectors are created, increasing the partition size by one sector to accommodate the extra FAT sector. To enable a single FAT sector, configure the `use_one_fat` option in `struct esp_vfs_fat_mount_config_t` (see :component_file:`fatfs/vfs/esp_vfs_fat.h`). Enabling this option allows the minimum partition size to be reduced to 32 KB.
+* The general formula for calculating the partition size for a wear-leveled partition is::
+
+    partition_size = Wear-levelling sectors * FLASH_SEC_SIZE + FATFS partition sectors * FAT_SEC_SIZE
+
+  Where:
+
+  - Wear-leveling sectors are fixed at 4
+  - FLASH_SEC_SIZE is 4096 bytes
+  - FATFS partition sectors include: 1 reserved sector + FAT sectors + root directory sectors + data sectors
+  - FAT_SEC_SIZE can be either 512 bytes or 4096 bytes, depending on the configuration
+
+* For read-only partitions without wear leveling enabled and a sector size of 512 bytes, the minimum partition size can be reduced to as low as 2 KB.
+
+Please refer :doc:`File System Considerations <../../api-guides/file-system-considerations>` for further details.
+
+
 High-level API Reference
 ------------------------