From a77705a938ead3e4257f864674ce8bad2400d848 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Tue, 11 Feb 2025 05:30:15 +0100 Subject: [PATCH 01/27] btrfs-progs: mkfs: update subvolume relatd error messages Unify the error message style a bit, print the relevant path if missing. Signed-off-by: David Sterba --- mkfs/main.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mkfs/main.c b/mkfs/main.c index 4191d04d7..f7b2aca9b 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -1142,7 +1142,7 @@ static int parse_subvolume(const char *path, struct list_head *subvols, if (subvol->is_default) { if (*has_default_subvol) { - error("default subvol can only be specified once"); + error("default subvolume can only be specified once"); free(subvol); return 1; } @@ -1453,25 +1453,25 @@ int BOX_MAIN(mkfs)(int argc, char **argv) struct rootdir_subvol *rds2; if (path_cat_out(path, source_dir, rds->dir)) { - error("path invalid"); + error("path invalid: %s", path); ret = 1; goto error; } if (!realpath(path, rds->full_path)) { - error("could not get canonical path to %s", rds->dir); + error("could not get canonical path: %s", rds->dir); ret = 1; goto error; } if (!path_exists(rds->full_path)) { - error("subvolume %s does not exist", rds->dir); + error("subvolume path does not exist: %s", rds->dir); ret = 1; goto error; } if (!path_is_dir(rds->full_path)) { - error("subvolume %s is not a directory", rds->dir); + error("subvolume is not a directory: %s", rds->dir); ret = 1; goto error; } @@ -1486,7 +1486,7 @@ int BOX_MAIN(mkfs)(int argc, char **argv) rds2 != rds; rds2 = list_next_entry(rds2, list)) { if (strcmp(rds2->full_path, rds->full_path) == 0) { - error("subvolume %s specified more than once", rds->dir); + error("subvolume specified more than once: %s", rds->dir); ret = 1; goto error; } From d0ed24d764ef138f1142f2d00e0986d75e49c9ae Mon Sep 17 00:00:00 2001 From: David Sterba Date: Tue, 11 Feb 2025 05:37:51 +0100 Subject: [PATCH 02/27] btrfs-progs: mkfs: print errno string instead of numeric values The preferred error message should have a prefix with problem description and then the errno description as we use the negative errno convention almost everywhere. - drop additional %d in the message if %m is present - replace %d with %m Signed-off-by: David Sterba --- mkfs/main.c | 64 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/mkfs/main.c b/mkfs/main.c index f7b2aca9b..faf2a3273 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -902,7 +902,8 @@ static int setup_quota_root(struct btrfs_fs_info *fs_info) } ret = btrfs_create_root(trans, fs_info, BTRFS_QUOTA_TREE_OBJECTID); if (ret < 0) { - error("failed to create quota root: %d (%m)", ret); + errno = -ret; + error("failed to create quota root: %m"); goto fail; } quota_root = fs_info->quota_root; @@ -914,7 +915,8 @@ static int setup_quota_root(struct btrfs_fs_info *fs_info) ret = btrfs_insert_empty_item(trans, quota_root, &path, &key, sizeof(*qsi)); if (ret < 0) { - error("failed to insert qgroup status item: %d (%m)", ret); + errno = -ret; + error("failed to insert qgroup status item: %m"); goto fail; } @@ -938,7 +940,8 @@ static int setup_quota_root(struct btrfs_fs_info *fs_info) /* Currently mkfs will only create one subvolume */ ret = insert_qgroup_items(trans, fs_info, BTRFS_FS_TREE_OBJECTID); if (ret < 0) { - error("failed to insert qgroup items: %d (%m)", ret); + errno = -ret; + error("failed to insert qgroup items: %m"); goto fail; } @@ -953,7 +956,8 @@ static int setup_quota_root(struct btrfs_fs_info *fs_info) if (simple) { ret = touch_root_subvol(fs_info); if (ret) { - error("failed to touch root dir for simple quota accounting %d (%m)", ret); + errno = -ret; + error("failed to touch root dir for simple quota accounting: %m"); goto fail; } } @@ -964,12 +968,15 @@ static int setup_quota_root(struct btrfs_fs_info *fs_info) */ ret = qgroup_verify_all(fs_info); if (ret < 0) { - error("qgroup rescan failed: %d (%m)", ret); + errno = -ret; + error("qgroup rescan failed: %m"); return ret; } ret = repair_qgroups(fs_info, &qgroup_repaired, true); - if (ret < 0) - error("failed to fill qgroup info: %d (%m)", ret); + if (ret < 0) { + errno = -ret; + error("failed to fill qgroup info: %m"); + } return ret; fail: btrfs_abort_transaction(trans, ret); @@ -1897,14 +1904,16 @@ int BOX_MAIN(mkfs)(int argc, char **argv) ret = create_metadata_block_groups(root, mixed, &allocation); if (ret) { - error("failed to create default block groups: %d", ret); + errno = -ret; + error("failed to create default block groups: %m"); goto error; } if (features.incompat_flags & BTRFS_FEATURE_INCOMPAT_RAID_STRIPE_TREE) { ret = setup_raid_stripe_tree_root(fs_info); if (ret < 0) { - error("failed to initialize raid-stripe-tree: %d (%m)", ret); + errno = -ret; + error("failed to initialize raid-stripe-tree: %m"); goto out; } } @@ -1919,21 +1928,24 @@ int BOX_MAIN(mkfs)(int argc, char **argv) ret = create_data_block_groups(trans, root, mixed, &allocation); if (ret) { - error("failed to create default data block groups: %d", ret); + errno = -ret; + error("failed to create default data block groups: %m"); goto error; } if (features.incompat_flags & BTRFS_FEATURE_INCOMPAT_EXTENT_TREE_V2) { ret = create_global_roots(trans, nr_global_roots); if (ret) { - error("failed to create global roots: %d", ret); + errno = -ret; + error("failed to create global roots: %m"); goto error; } } ret = make_root_dir(trans, root); if (ret) { - error("failed to setup the root directory: %d", ret); + errno = -ret; + error("failed to setup the root directory: %m"); goto error; } @@ -1976,8 +1988,9 @@ int BOX_MAIN(mkfs)(int argc, char **argv) prepare_ctx[i].file, dev_byte_count, sectorsize, sectorsize, sectorsize); if (ret) { - error("unable to add %s to filesystem: %d", - prepare_ctx[i].file, ret); + errno = -ret; + error("unable to add %s to filesystem: %m", + prepare_ctx[i].file); goto error; } if (bconf.verbose >= LOG_INFO) { @@ -1996,7 +2009,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv) ret = create_raid_groups(trans, root, data_profile, metadata_profile, mixed, &allocation); if (ret) { - error("unable to create raid groups: %d", ret); + errno = -ret; + error("unable to create raid groups: %m"); goto out; } @@ -2031,7 +2045,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv) ret = btrfs_make_subvolume(trans, BTRFS_DATA_RELOC_TREE_OBJECTID, false); if (ret) { - error("unable to create data reloc tree: %d", ret); + errno = -ret; + error("unable to create data reloc tree: %m"); goto out; } @@ -2065,7 +2080,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv) &subvols, compression, compression_level); if (ret) { - error("error while filling filesystem: %d", ret); + errno = -ret; + error("error while filling filesystem: %m"); btrfs_abort_transaction(trans, ret); goto out; } @@ -2087,8 +2103,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv) ret = btrfs_mkfs_shrink_fs(fs_info, &shrink_size, shrink_rootdir); if (ret < 0) { - error("error while shrinking filesystem: %d", - ret); + errno = -ret; + error("error while shrinking filesystem: %m"); goto out; } } else { @@ -2103,7 +2119,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv) ret = cleanup_temp_chunks(fs_info, &allocation, data_profile, metadata_profile, metadata_profile); if (ret < 0) { - error("failed to cleanup temporary chunks: %d", ret); + errno = -ret; + error("failed to cleanup temporary chunks: %m"); goto out; } @@ -2111,7 +2128,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv) features.incompat_flags & BTRFS_FEATURE_INCOMPAT_SIMPLE_QUOTA) { ret = setup_quota_root(fs_info); if (ret < 0) { - error("failed to initialize quota: %d (%m)", ret); + errno = -ret; + error("failed to initialize quota: %m"); goto out; } } @@ -2184,8 +2202,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv) if (!ret && close_ret) { ret = close_ret; - error("failed to close ctree, the filesystem may be inconsistent: %d", - ret); + errno = -ret; + error("failed to close ctree, filesystem may be inconsistent: %m"); } btrfs_close_all_devices(); From 83c3bc48e4450e99c91b99a986088c1c04e09eb0 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Tue, 11 Feb 2025 06:19:10 +0100 Subject: [PATCH 03/27] btrfs-progs: mkfs: fix printing of subvolumes in the summary The subvolumes created during mkfs are not printed in the summary because btrfs_mkfs_fill_dir() deletes them from the list as they get created. Signed-off-by: David Sterba --- mkfs/main.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mkfs/main.c b/mkfs/main.c index faf2a3273..f99af624f 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -2076,6 +2076,12 @@ int BOX_MAIN(mkfs)(int argc, char **argv) pretty_size_mode(compression_level, UNITS_RAW) : ""); + /* Print subvolumes now as btrfs_mkfs_fill_dir() deletes the list. */ + list_for_each_entry(rds, &subvols, list) { + pr_verbose(LOG_DEFAULT, " Subvolume: %s\n", + rds->full_path); + } + ret = btrfs_mkfs_fill_dir(trans, source_dir, root, &subvols, compression, compression_level); @@ -2093,11 +2099,6 @@ int BOX_MAIN(mkfs)(int argc, char **argv) goto out; } - list_for_each_entry(rds, &subvols, list) { - pr_verbose(LOG_DEFAULT, " Subvolume: %s\n", - rds->full_path); - } - if (shrink_rootdir) { pr_verbose(LOG_DEFAULT, " Shrink: yes\n"); ret = btrfs_mkfs_shrink_fs(fs_info, &shrink_size, From b34ba8444084fbc530bd1b3f8f8fe025718612c8 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Tue, 11 Feb 2025 06:12:38 +0100 Subject: [PATCH 04/27] btrfs-progs: mkfs: print type of subvolume in the listing Print indicators in the summary if the subvolume is read-write, read-only or default: $ mkfs.btrfs --subvol ro:subvolro --subvol rw:subvolrw --subvol default-ro:defaultro --rootdir /rootdir/path img ... Rootdir from: /rootdir/path Compress: no Subvolume (rw): subvolrw Subvolume (ro): subvolro Subvolume (dro): defaultro ... The path is relative to the rootdir path and may not be a subvolume in the source directory so drop the rootdir as this may be confusing. Signed-off-by: David Sterba --- mkfs/main.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mkfs/main.c b/mkfs/main.c index f99af624f..3e5e18c3b 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -2078,8 +2078,11 @@ int BOX_MAIN(mkfs)(int argc, char **argv) /* Print subvolumes now as btrfs_mkfs_fill_dir() deletes the list. */ list_for_each_entry(rds, &subvols, list) { - pr_verbose(LOG_DEFAULT, " Subvolume: %s\n", - rds->full_path); + pr_verbose(LOG_DEFAULT, " Subvolume (%s%s): %s%s\n", + rds->is_default ? "d" : "", + rds->readonly ? "ro" : "rw", + rds->is_default ? "" : " ", + rds->dir); } ret = btrfs_mkfs_fill_dir(trans, source_dir, root, From 8f580adaf8bc5138f9de42dbc03ae748744dd9be Mon Sep 17 00:00:00 2001 From: David Sterba Date: Tue, 11 Feb 2025 06:40:38 +0100 Subject: [PATCH 05/27] btrfs-progs: docs: update mkfs.btrfs manual page Fix missing or wrong formatting, enhance descriptions or add more references. Signed-off-by: David Sterba --- Documentation/mkfs.btrfs.rst | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/Documentation/mkfs.btrfs.rst b/Documentation/mkfs.btrfs.rst index f58c2fe62..329d01145 100644 --- a/Documentation/mkfs.btrfs.rst +++ b/Documentation/mkfs.btrfs.rst @@ -150,6 +150,9 @@ OPTIONS Populate the toplevel subvolume with files from *rootdir*. This does not require root permissions to write the new files or to mount the filesystem. + Directories can be created as subvolumes, see also option *--subvol*. + Hardlinks are detected and created in the filesystem image. + .. note:: This option may enlarge the image or file to ensure it's big enough to contain the files from *rootdir*. Since version 4.14.1 the filesystem size is @@ -160,6 +163,10 @@ OPTIONS *no* (the default), *zstd*, *lzo* or *zlib*. The optional value *level* is a compression level, 1..15 for *zstd*, 1..9 for *zlib*. + It is recommended to use the highest level to achieve maximum space savings. + Compression at mkfs time is not as constrained as in kernel where it's + desirable to use the less CPU load. Otherwise the default level is 3. + As with the kernel, :command:`mkfs.btrfs` won't write compressed extents when they would be larger than the uncompressed versions, and will set file attribute *NOCOMPRESS* if its beginning is found to be incompressible. @@ -173,7 +180,7 @@ OPTIONS directory. The option *--rootdir* must also be specified, and *subdir* must be an existing subdirectory within it. This option can be specified multiple times. - *type* is an optional additional modifier. Valid choices are: + The *type* is an optional additional modifier. Valid choices are: * *default*: create as default subvolume * *ro*: create as read-only subvolume @@ -183,14 +190,16 @@ OPTIONS Only one of *default* and *default-ro* may be specified. If you wish to create a subvolume with a name containing a colon and you don't - want this to be parsed as containing a modifier, you can prefix the path with `./`: + want this to be parsed as containing a modifier, you can prefix the path with :file:`./`: .. code-block:: bash $ mkfs.btrfs --rootdir dir --subvol ./ro:subdir /dev/loop0 - If there are hard links inside *rootdir* and *subdir* will split the - subvolumes, like the following case:: + If there are hardlinks inside *rootdir* and *subdir* will split the + subvolumes, like the following case: + + .. code-block:: none rootdir/ |- hardlink1 @@ -198,9 +207,9 @@ OPTIONS |- subdir/ <- will be a subvolume |- hardlink3 - In that case we cannot create `hardlink3` as hardlinks of - `hardlink1` and `hardlink2` because hardlink3 will be inside a new - subvolume. + In that case we cannot create :file:`hardlink3` as hardlinks of + :file:`hardlink1` and :file:`hardlink2` because :file:`hardlink3` will + be inside a new subvolume. --shrink Shrink the filesystem to its minimal size, only works with *--rootdir* option. From 06e9c955deb03a96e4dbc7ccf0b055e547768d39 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Tue, 11 Feb 2025 18:59:30 +0100 Subject: [PATCH 06/27] btrfs-progs: ci: update Dockerfile CMD syntax Docker warns about CMD syntax, which is harmless in our case but let's fix the warning by using the json (quoted strings in array) syntax. JSONArgsRecommended: JSON arguments recommended for CMD to prevent unintended behavior related to OS signals (line 29) Explained in https://docs.docker.com/reference/build-checks/json-args-recommended . Signed-off-by: David Sterba --- ci/images/ci-centos-7-x86_64/Dockerfile | 2 +- ci/images/ci-centos-8-x86_64/Dockerfile | 2 +- ci/images/ci-musl-i386/Dockerfile | 2 +- ci/images/ci-musl-x86_64/Dockerfile | 2 +- ci/images/ci-openSUSE-Leap-15.3-x86_64/Dockerfile | 2 +- ci/images/ci-openSUSE-Leap-15.6-x86_64/Dockerfile | 2 +- ci/images/ci-openSUSE-tumbleweed-x86_64/Dockerfile | 2 +- ci/images/ci-rockylinux-9-x86_64/Dockerfile | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ci/images/ci-centos-7-x86_64/Dockerfile b/ci/images/ci-centos-7-x86_64/Dockerfile index 6b49055dd..c4e3bcd5e 100644 --- a/ci/images/ci-centos-7-x86_64/Dockerfile +++ b/ci/images/ci-centos-7-x86_64/Dockerfile @@ -29,7 +29,7 @@ COPY ./test-build . COPY ./run-tests . COPY ./devel.tar.gz . -CMD ./test-build devel --disable-documentation --disable-libudev +CMD ["./test-build", "devel", "--disable-documentation", "--disable-libudev"] # Continue with: # cd /tmp diff --git a/ci/images/ci-centos-8-x86_64/Dockerfile b/ci/images/ci-centos-8-x86_64/Dockerfile index b10f51126..d36ad510f 100644 --- a/ci/images/ci-centos-8-x86_64/Dockerfile +++ b/ci/images/ci-centos-8-x86_64/Dockerfile @@ -32,7 +32,7 @@ COPY ./test-build . COPY ./run-tests . COPY ./devel.tar.gz . -CMD ./test-build devel --disable-documentation --disable-libudev --disable-zoned +CMD ["./test-build", "devel", "--disable-documentation", "--disable-libudev", "--disable-zoned"] # Continue with: # cd /tmp diff --git a/ci/images/ci-musl-i386/Dockerfile b/ci/images/ci-musl-i386/Dockerfile index 912d2beb6..f6248d4f6 100644 --- a/ci/images/ci-musl-i386/Dockerfile +++ b/ci/images/ci-musl-i386/Dockerfile @@ -16,4 +16,4 @@ RUN apk add wget COPY ./test-build . COPY ./devel.tar.gz . -CMD ./test-build devel --disable-documentation --disable-backtrace --disable-libudev +CMD ["./test-build", "devel", "--disable-documentation", "--disable-backtrace", "--disable-libudev"] diff --git a/ci/images/ci-musl-x86_64/Dockerfile b/ci/images/ci-musl-x86_64/Dockerfile index 912d2beb6..f6248d4f6 100644 --- a/ci/images/ci-musl-x86_64/Dockerfile +++ b/ci/images/ci-musl-x86_64/Dockerfile @@ -16,4 +16,4 @@ RUN apk add wget COPY ./test-build . COPY ./devel.tar.gz . -CMD ./test-build devel --disable-documentation --disable-backtrace --disable-libudev +CMD ["./test-build", "devel", "--disable-documentation", "--disable-backtrace", "--disable-libudev"] diff --git a/ci/images/ci-openSUSE-Leap-15.3-x86_64/Dockerfile b/ci/images/ci-openSUSE-Leap-15.3-x86_64/Dockerfile index 5fecce0fb..49d00d5d2 100644 --- a/ci/images/ci-openSUSE-Leap-15.3-x86_64/Dockerfile +++ b/ci/images/ci-openSUSE-Leap-15.3-x86_64/Dockerfile @@ -26,7 +26,7 @@ COPY ./run-tests . COPY ./devel.tar.gz . # The blkzoned.h exists but blk_zone.capacity is missing, disable zoned mode explicitly -CMD ./test-build devel --disable-documentation --disable-zoned +CMD ["./test-build", "devel", "--disable-documentation", "--disable-zoned"] # Continue with: # cd /tmp diff --git a/ci/images/ci-openSUSE-Leap-15.6-x86_64/Dockerfile b/ci/images/ci-openSUSE-Leap-15.6-x86_64/Dockerfile index df8af2466..2a4fedecf 100644 --- a/ci/images/ci-openSUSE-Leap-15.6-x86_64/Dockerfile +++ b/ci/images/ci-openSUSE-Leap-15.6-x86_64/Dockerfile @@ -26,7 +26,7 @@ COPY ./run-tests . COPY ./devel.tar.gz . # The blkzoned.h exists but blk_zone.capacity is missing, disable zoned mode explicitly -CMD ./test-build devel --disable-documentation --disable-zoned +CMD ["./test-build", "devel", "--disable-documentation", "--disable-zoned"] # Continue with: # cd /tmp diff --git a/ci/images/ci-openSUSE-tumbleweed-x86_64/Dockerfile b/ci/images/ci-openSUSE-tumbleweed-x86_64/Dockerfile index 554745de6..d6ecde420 100644 --- a/ci/images/ci-openSUSE-tumbleweed-x86_64/Dockerfile +++ b/ci/images/ci-openSUSE-tumbleweed-x86_64/Dockerfile @@ -33,7 +33,7 @@ COPY ./test-build . COPY ./run-tests . COPY ./devel.tar.gz . -CMD ./test-build devel --disable-documentation +CMD ["./test-build", "devel", "--disable-documentation"] # Continue with: # cd /tmp diff --git a/ci/images/ci-rockylinux-9-x86_64/Dockerfile b/ci/images/ci-rockylinux-9-x86_64/Dockerfile index 7c1651caf..3328dfdbb 100644 --- a/ci/images/ci-rockylinux-9-x86_64/Dockerfile +++ b/ci/images/ci-rockylinux-9-x86_64/Dockerfile @@ -29,7 +29,7 @@ COPY ./test-build . COPY ./run-tests . COPY ./devel.tar.gz . -CMD ./test-build devel --disable-documentation +CMD ["./test-build", "devel", "--disable-documentation"] # Continue with: # cd /tmp From 800d7b0fe63062847b3b779a4d7d20f9af55d219 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Tue, 11 Feb 2025 18:18:07 +0000 Subject: [PATCH 07/27] btrfs-progs: ci: add workflow for CodeQL Enable CodeQL code scanning checks. For that the automatically detected build steps don't work as the documentation is expected to build. Add the manual steps. Signed-off-by: David Sterba --- .github/workflows/codeql.yml | 105 +++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..e2700d27a --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,105 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL Advanced" + +on: + push: + branches: [ "devel" ] + pull_request: + branches: [ "devel" ] + schedule: + - cron: '37 22 * * 0' + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners (GitHub.com only) + # Consider using larger runners or machines with greater resources for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + permissions: + # required for all workflows + security-events: write + + # required to fetch internal or private CodeQL packs + packages: read + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + include: + - language: c-cpp + build-mode: manual + - language: python + build-mode: none + # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' + # Use `c-cpp` to analyze code written in C, C++ or both + # Use 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, + # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. + # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how + # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Add any setup steps before running the `github/codeql-action/init` action. + # This includes steps like installing compilers or runtimes (`actions/setup-node` + # or others). This is typically only required for manual builds. + # - name: Setup runtime (example) + # uses: actions/setup-example@v1 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + - name: Update package cache + run: sudo apt-get update -qqq + - name: Add more build dependencies + run: sudo apt-get install -y liblzo2-dev libzstd-dev libblkid-dev uuid-dev zlib1g-dev libext2fs-dev e2fsprogs libudev-dev + + # If the analyze step fails for one of the languages you are analyzing with + # "We were unable to automatically build your code", modify the matrix above + # to set the build mode to "manual" for that language. Then modify this step + # to build your code. + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + - if: matrix.build-mode == 'manual' + shell: bash + run: | + echo 'If you are using a "manual" build mode for one or more of the' \ + 'languages you are analyzing, replace this with the commands to build' \ + 'your code, for example:' + ./autogen.sh + ./configure --disable-documentation --enable-experimental + make -j 4 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" From 66094893747edf5adc640f2f51c2f42328825899 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Tue, 11 Feb 2025 23:35:38 +0100 Subject: [PATCH 08/27] btrfs-progs: README: add status badge of codeql workflow [ci skip] Signed-off-by: David Sterba --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 055c11e88..e93d29c6b 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Btrfs-progs [![codecov](https://codecov.io/gh/kdave/btrfs-progs/branch/coverage-test/graph/badge.svg?token=fhLI8V9s0k)](https://codecov.io/gh/kdave/btrfs-progs) [![static](https://github.com/kdave/btrfs-progs/actions/workflows/artifacts-static-build.yml/badge.svg)](https://github.com/kdave/btrfs-progs/actions/workflows/artifacts-static-build.yml) [![release](https://github.com/kdave/btrfs-progs/actions/workflows/ci-build-test.yml/badge.svg)](https://github.com/kdave/btrfs-progs/actions/workflows/ci-build-test.yml) +[![codeql](https://github.com/kdave/btrfs-progs/actions/workflows/codeql.yml/badge.svg)](https://github.com/kdave/btrfs-progs/actions/workflows/codeql.yml) Userspace utilities to manage btrfs filesystems. License: GPLv2. From 3d8ab67fbd6c9560303f8d82adfcaec7ffe63127 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 12 Feb 2025 00:01:44 +0100 Subject: [PATCH 09/27] btrfs-progs: fix UNITS_DECIMAL and UNITS_NEGATIVE bit definitions The values of UNITS_ modes are supposed to be unique and non-overlapping. Due to wrong definition this was not true for UNITS_DECIMAL which set 2 bits. This breaks code that strictly depends on the uniqueness of the bits and not on the order of processing of the constants. Signed-off-by: David Sterba --- common/units.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/units.h b/common/units.h index d93977ff2..4a638e7f0 100644 --- a/common/units.h +++ b/common/units.h @@ -31,9 +31,9 @@ #define UNITS_TBYTES (5) #define UNITS_RAW (1U << UNITS_MODE_SHIFT) #define UNITS_BINARY (2U << UNITS_MODE_SHIFT) -#define UNITS_DECIMAL (3U << UNITS_MODE_SHIFT) +#define UNITS_DECIMAL (4U << UNITS_MODE_SHIFT) /* Interpret the u64 value as s64 */ -#define UNITS_NEGATIVE (4U << UNITS_MODE_SHIFT) +#define UNITS_NEGATIVE (8U << UNITS_MODE_SHIFT) #define UNITS_MODE_MASK ((1U << UNITS_MODE_SHIFT) - 1) #define UNITS_MODE_SHIFT (8) #define UNITS_HUMAN_BINARY (UNITS_BINARY) From 083d2263555c132b72c9b9734d793c7895f93dce Mon Sep 17 00:00:00 2001 From: David Sterba Date: Tue, 11 Feb 2025 23:44:31 +0100 Subject: [PATCH 10/27] btrfs-progs: scrub status: fix rate units There's a special case when scrub rate is printed, the device sizes and rate use different units. The size can be in terabytes while the rate can be in hundreds of megabytes (contemporary 10-20T disks, 250MB/s). The sizes use what is set on command line (or human readable by default), while the rate is always human readable with exception to the option --raw to provide a way to print the exact numbers without any conversions. This got broken in commit ec3c8428590e90 ("btrfs-progs: scrub status: with --si, show rate in metric units") that forced the command line mode to the rate as well. Instead of that we need to detect the SI/IEC mode and set it to the human readable format of rate. Signed-off-by: David Sterba --- cmds/scrub.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cmds/scrub.c b/cmds/scrub.c index 23af16280..b1d2f731b 100644 --- a/cmds/scrub.c +++ b/cmds/scrub.c @@ -204,9 +204,10 @@ static void print_scrub_summary(struct btrfs_scrub_progress *p, struct scrub_sta pr_verbose(LOG_DEFAULT, "Total to scrub: %s\n", pretty_size_mode(bytes_total, unit_mode)); } + /* * Rate and size units are disproportionate so they are affected only - * by --raw, otherwise it's human readable + * by --raw, otherwise it's human readable (respecting the SI or IEC mode). */ if (unit_mode == UNITS_RAW) { pr_verbose(LOG_DEFAULT, "Rate: %s/s", @@ -218,11 +219,16 @@ static void print_scrub_summary(struct btrfs_scrub_progress *p, struct scrub_sta pr_verbose(LOG_DEFAULT, " (some device limits set)"); pr_verbose(LOG_DEFAULT, "\n"); } else { + unsigned int mode = UNITS_HUMAN_DECIMAL; + + if (unit_mode & UNITS_BINARY) + mode = UNITS_HUMAN_BINARY; + pr_verbose(LOG_DEFAULT, "Rate: %s/s", - pretty_size_mode(bytes_per_sec, unit_mode)); + pretty_size_mode(bytes_per_sec, mode)); if (limit > 1) pr_verbose(LOG_DEFAULT, " (limit %s/s)", - pretty_size_mode(limit, unit_mode)); + pretty_size_mode(limit, mode)); else if (limit == 1) pr_verbose(LOG_DEFAULT, " (some device limits set)"); pr_verbose(LOG_DEFAULT, "\n"); From 4382c2fbbeefdc20bbb0319c7907aedb28a3b8f9 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 12 Feb 2025 02:26:03 +0100 Subject: [PATCH 11/27] btrfs-progs: docs: update json API Add currently implemented types and update recommendations. Signed-off-by: David Sterba --- Documentation/dev/dev-json.rst | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/Documentation/dev/dev-json.rst b/Documentation/dev/dev-json.rst index 09d853313..498381058 100644 --- a/Documentation/dev/dev-json.rst +++ b/Documentation/dev/dev-json.rst @@ -3,12 +3,17 @@ JSON output Supported types: +* any: a valid *printf* format, parameters to *fmt_print()* must match the number (and are not validated) * number: ``%llu`` * string: ``%s`` -* string: ``str`` (escaped special characters) -* bool: ``bool`` (unquoted native json value) -* qgroupid: ``qgroupid`` (split to 48/16 for level/subvolid) -* size: ``size`` +* string: ``str`` - backslash escaped special characters (0x08, 0x08, 0x0a, 0x0c, 0x0d, 0x0b), + the rest of range from *0x00* to *0x1f* as *\\uXXXX* encoding +* bool: ``bool`` - unquoted native json values *true* or *false* +* qgroupid: ``qgroupid`` - split to 48/16 for level/subvolid +* size: ``size`` - size with SI/IEC size suffix +* size: ``size-or-none`` - same as *size* but for 0 it's *none* +* UUID: ``uuid`` - if all zeros then *null* (native json), or properly formatted UUID string +* date + time: ``date-time`` - timestamp formatted as *YYYY-MM-DD HH:MM:SS TIMEZONE* Commands that support json output --------------------------------- @@ -23,12 +28,18 @@ Commands that support json output Recommendations --------------- +Keys and formatting are defined as an array of *struct rowspec*. + * key names - * should be unified if they mean the same thing + * should be unified with the printed value if they mean the same thing * not abbreviated (e.g. *generation* instead of *gen*) * referring to existing and well known names (qgroupid, devid, ...) + * using ``-`` as word separator, or ``_`` if it's better to keep the same name of the value * values * numbers without suffix or other transformation, i.e. no *KiB* + * formatted by the types if possible + * any *printf* format is possible but should be avoided or a new type should + be defined * printing more data about an item is better than printing less, assuming the filtering is done on the user side * structure of json output may not reflect the way it's printed in plain text, @@ -36,3 +47,5 @@ Recommendations * if plain and json output roughly follow the same style, e.g. line oriented that is easy to transform to a map, then both outputs should use the same rowspec +* one value can be printed by multiple rowspecs that may do different + formatting depending on the context From c5c94565331aae454d82fa035dba2cc10f4af997 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 12 Feb 2025 02:46:16 +0100 Subject: [PATCH 12/27] btrfs-progs: docs: fix typos Thanks to hunspell. [ci skip] Signed-off-by: David Sterba --- Documentation/DocConventions.rst | 2 +- Documentation/Kernel-by-version.rst | 16 ++++++++-------- Documentation/btrfs-filesystem.rst | 2 +- Documentation/btrfs-ioctl.rst | 4 ++-- Documentation/btrfs-man5.rst | 2 +- Documentation/ch-sysfs.rst | 4 ++-- Documentation/dev/Developer-s-FAQ.rst | 6 +++--- Documentation/dev/Development-notes.rst | 2 +- Documentation/dev/dev-btrees.rst | 2 +- Documentation/mkfs.btrfs.rst | 2 +- 10 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Documentation/DocConventions.rst b/Documentation/DocConventions.rst index 96a96c617..955b12e57 100644 --- a/Documentation/DocConventions.rst +++ b/Documentation/DocConventions.rst @@ -67,7 +67,7 @@ Referencing: - generic links can use the free form link syntax with description ```Link text `__`` (note the double underscore, this is *anonymous* link and does not create a reference) - or plain link that will auto-render to a clikable link https://example.com + or plain link that will auto-render to a clickable link https://example.com - in manual pages: always use full link as it's meant to be read in terminal output and must allow copy&paste diff --git a/Documentation/Kernel-by-version.rst b/Documentation/Kernel-by-version.rst index 980946c5d..f2dd36d18 100644 --- a/Documentation/Kernel-by-version.rst +++ b/Documentation/Kernel-by-version.rst @@ -853,7 +853,7 @@ Pull requests: `v5.4-rc7 `__, `v5.4-rc8 `__ -- tree checker: adde sanity checks for tree items, extent items, and references +- tree checker: added sanity checks for tree items, extent items, and references - deprecated subvolume creation mode BTRFS_SUBVOL_CREATE_ASYNC - qgroup relation deletion tries harder, orphan entries are removed too - space handling improvements (ticket reservations, flushing, overcommit logic) @@ -1023,7 +1023,7 @@ Pull requests: Highlights: - performance improvements in fsync (dbench workload: higher throughput, lower latency) -- sysfs exports current exclusive operataion (balance, resize, device add/del/...) +- sysfs exports current exclusive operation (balance, resize, device add/del/...) - sysfs exports supported send stream version Core: @@ -1083,7 +1083,7 @@ Features: Core changes: -- subpage block size support peparations +- subpage block size support preparations Fixes: @@ -1434,7 +1434,7 @@ Qgroup: - limits are shared upon snapshot - allow to remove qgroup which has parent but no child - fix status of qgroup consistency after rescan -- fix quota status bits after dsiabling +- fix quota status bits after disabling - mark qgroups inconsistent after assign/delete actions - code cleanups @@ -1475,7 +1475,7 @@ Fixes: ^^^^^^^^^^^^^^ - send fixes: cloning, sending with parent -- improved handling of framgented space using bitmaps +- improved handling of fragmented space using bitmaps - new mount option for debugging: fragment=data|metadata|all - updated balance filters: limit, stripes, usage - more bugfixes and cleanups @@ -1501,7 +1501,7 @@ Fixes: writes (now does: nologreplay) - default inline limit is now 2048 (instead of page size, usually 4096) -- /dev/btrfs-control now understands the GET_SUPPORTE_FEATURES ioctl +- /dev/btrfs-control now understands the GET_SUPPORTED_FEATURES ioctl - get rid of harmless message "''could not find root %llu''" - preparatory work for subpage-blocksize patchset - fix bug when using overlayfs @@ -1574,7 +1574,7 @@ Fixes: ^^^^^^^^^^^^^^^ - new tracepoints: file item -- fix qgoup accounting when inode_cache is in use +- fix qgroup accounting when inode_cache is in use - fix incorrect number report in stat::t_blocks under certain conditions - raid56 fixes: @@ -1877,7 +1877,7 @@ Fixes: ^^^^^^^^^^^^^^^ * ''fiemap'' exports information about shared extents -* bugfix and stability foucsed release +* bugfix and stability focused release 3.14 (Mar 2014) ^^^^^^^^^^^^^^^ diff --git a/Documentation/btrfs-filesystem.rst b/Documentation/btrfs-filesystem.rst index b036ae5ea..923032022 100644 --- a/Documentation/btrfs-filesystem.rst +++ b/Documentation/btrfs-filesystem.rst @@ -140,7 +140,7 @@ defragment [options] | [|...] logic. Reasonable values are from tens to hundreds of megabytes. --step SIZE - Perform defragmention in the range in SIZE steps and flush (*-f*) after each one. + Perform defragmentation in the range in SIZE steps and flush (*-f*) after each one. The range is default (the whole file) or given by *-s* and *-l*, split into the steps or done in one go if the step is larger. Minimum range size is 256KiB. diff --git a/Documentation/btrfs-ioctl.rst b/Documentation/btrfs-ioctl.rst index 2a03ea375..a3d1353e9 100644 --- a/Documentation/btrfs-ioctl.rst +++ b/Documentation/btrfs-ioctl.rst @@ -550,7 +550,7 @@ BTRFS_IOC_ADD_DEV Add a given block device to the filesystem. Unlike the command :command:`btrfs device add` there's are no safety checks (like existence of another filesystem on the -device), device preparataion (like TRIM or zone reset), so use it with care. +device), device preparation (like TRIM or zone reset), so use it with care. This is a filesystem-exclusive operation and it will fail if there's another one already running, with one exception, when there's a paused balance. @@ -968,7 +968,7 @@ Use cases (:ref:`definition of constants`): * - ioctl args - :ref:`struct btrfs_ioctl_subvol_wait` * - args.subvolid - - Depending on the mode, the numeric id of subolume to wait for, or + - Depending on the mode, the numeric id of subvolume to wait for, or the one queried by *PEEK* modes * - args.mode - mode of operation described above diff --git a/Documentation/btrfs-man5.rst b/Documentation/btrfs-man5.rst index 56f9c939d..fb85c93ff 100644 --- a/Documentation/btrfs-man5.rst +++ b/Documentation/btrfs-man5.rst @@ -220,7 +220,7 @@ SYSFS INTERFACE .. include:: ch-sysfs.rst -.. _man-btrfs5-fileysstem-exclusive-operations: +.. _man-btrfs5-filesystem-exclusive-operations: FILESYSTEM EXCLUSIVE OPERATIONS ------------------------------- diff --git a/Documentation/ch-sysfs.rst b/Documentation/ch-sysfs.rst index f58db400f..cc5193f80 100644 --- a/Documentation/ch-sysfs.rst +++ b/Documentation/ch-sysfs.rst @@ -72,7 +72,7 @@ exclusive_operation Shows the running exclusive operation. Check section - :ref:`FILESYSTEM EXCLUSIVE OPERATIONS` + :ref:`FILESYSTEM EXCLUSIVE OPERATIONS` for details. generation @@ -216,7 +216,7 @@ scrub_speed_max (RW, since: 5.14) Shows the scrub speed limit for this device. The unit is Bytes/s. - 0 means no limit. The value can be set but is not persisent. + 0 means no limit. The value can be set but is not persistent. writeable (RO, since: 5.6) diff --git a/Documentation/dev/Developer-s-FAQ.rst b/Documentation/dev/Developer-s-FAQ.rst index 3b33207d1..f258a6cca 100644 --- a/Documentation/dev/Developer-s-FAQ.rst +++ b/Documentation/dev/Developer-s-FAQ.rst @@ -64,7 +64,7 @@ candidate for a CC: of the bugreport. **Do**: talk to the maintainer if he forgot to add this tag to the final patch. Reviews do take time and the patches land in various branches early after -they're sent to the mailingslist for testing, but the reviews are always +they're sent to the mailinglist for testing, but the reviews are always welcome. **Do**: collect the Reviewed-by tags for patches that get resent unchanged e.g. @@ -236,7 +236,7 @@ Good practices, contribution hints - be patient if your patch is not accepted immediately, try to send a gentle ping if there's a significant time without any action - if you want to start contributing but are not sure about how to do that, - lurk in the mailingist or on the IRC channel + lurk in the mailinglist or on the IRC channel - every patch should implement one thing -- this is vaguely defined, you may receive comments about patch splitting or merging with other - every patch must be compilable when applied, possibly with all related @@ -451,7 +451,7 @@ copyright holders of changes in a given file. The code is usually heavily changed over time in smaller portions, slowly morphing into something that does not resemble the original code anymore though it shares a lot of the core ideas and implemented logic. A copyright notice by a company that does not exist -anymore from 10 years ago is a clear example of uselesness for the developers. +anymore from 10 years ago is a clear example of uselessness for the developers. When code is moved verbatim from a file to another file, in the new file it appears to be contributed by a single author while it is in most cases code diff --git a/Documentation/dev/Development-notes.rst b/Documentation/dev/Development-notes.rst index 76c492cae..65b7af4e4 100644 --- a/Documentation/dev/Development-notes.rst +++ b/Documentation/dev/Development-notes.rst @@ -120,7 +120,7 @@ There are several ways how to react to the unexpected conditions: This would cause the filesystem to be flipped read-only to prevent further corruption. - Additionally call trace would be dumpped for the first btrfs_abort_transaction() + Additionally call trace would be dumped for the first btrfs_abort_transaction() call site. - ASSERT() diff --git a/Documentation/dev/dev-btrees.rst b/Documentation/dev/dev-btrees.rst index 1a710f881..83e090fc1 100644 --- a/Documentation/dev/dev-btrees.rst +++ b/Documentation/dev/dev-btrees.rst @@ -114,7 +114,7 @@ Log Root Tree UUID Tree --------- -The tree storesc correspondence between UUIDs and subvolumes. Used for +The tree stores correspondence between UUIDs and subvolumes. Used for quick lookup during send. diff --git a/Documentation/mkfs.btrfs.rst b/Documentation/mkfs.btrfs.rst index 329d01145..9d8cfe955 100644 --- a/Documentation/mkfs.btrfs.rst +++ b/Documentation/mkfs.btrfs.rst @@ -614,7 +614,7 @@ hopefully not destroying both copies of particular data in case of DUP. The wear levelling techniques can also lead to reduced redundancy, even if the device does not do any deduplication. The controllers may put data written in -a short timespan into the same physical storage unit (cell, block etc). In case +a short time span into the same physical storage unit (cell, block etc). In case this unit dies, both copies are lost. BTRFS does not add any artificial delay between metadata writes. From 9e7853ce94de5cb6c2f2e8a8eba2ab39c0a089d4 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 12 Feb 2025 03:18:22 +0100 Subject: [PATCH 13/27] btrfs-progs: docs: update sysfs descriptions Add sections for each directory so it's more visible where the sections start and end. Update formatting, enhance descriptions. Signed-off-by: David Sterba --- Documentation/ch-sysfs.rst | 104 +++++++++++++++++++++++++++---------- 1 file changed, 77 insertions(+), 27 deletions(-) diff --git a/Documentation/ch-sysfs.rst b/Documentation/ch-sysfs.rst index cc5193f80..1da273993 100644 --- a/Documentation/ch-sysfs.rst +++ b/Documentation/ch-sysfs.rst @@ -5,26 +5,31 @@ The top level path is :file:`/sys/fs/btrfs/`, and the main directory layout is t ============================= =================================== ======== Relative Path Description Version ============================= =================================== ======== -features/ All supported features 3.14+ -/ Mounted fs UUID 3.14+ -/allocation/ Space allocation info 3.14+ -/features/ Features of the filesystem 3.14+ -/devices// Symlink to each block device sysfs 5.6+ -/devinfo// Btrfs specific info for each device 5.6+ -/qgroups/ Global qgroup info 5.9+ -/qgroups/_/ Info for each qgroup 5.9+ -/discard/ Discard stats and tunables 6.1+ +features/ All supported features 3.14 +/ Mounted fs UUID 3.14 +/allocation/ Space allocation info 3.14 +/bdi/ Backing device info (writeback) 5.9 +/devices// Symlink to each block device sysfs 5.6 +/devinfo// Btrfs specific info for each device 5.6 +/discard/ Discard stats and tunables 6.1 +/features/ Features of the filesystem 3.14 +/qgroups/ Global qgroup info 5.9 +/qgroups/_/ Info for each qgroup 5.9 ============================= =================================== ======== For :file:`/sys/fs/btrfs/features/` directory, each file means a supported feature -for the current kernel. +of the current kernel. Most files have value 0. Otherwise it depends on the file, +value *1* typically means the feature can be turned on a mounted filesystem. For :file:`/sys/fs/btrfs//features/` directory, each file means an enabled -feature for the mounted filesystem. +feature on the mounted filesystem. -The features shares the same name in section +The features share the same name in section :ref:`FILESYSTEM FEATURES`. +UUID +^^^^ + Files in :file:`/sys/fs/btrfs//` directory are: bg_reclaim_threshold @@ -88,7 +93,7 @@ label metadata_uuid (RO, since: 5.0) - Shows the metadata uuid of the mounted filesystem. + Shows the metadata UUID of the mounted filesystem. Check `metadata_uuid` feature for more details. nodesize @@ -107,20 +112,24 @@ read_policy (RW, since: 5.11) Shows the current balance policy for reads. - Currently only "pid" (balance using pid value) is supported. + Currently only ``pid`` (balance using the process id (pid) value) is + supported. More balancing policies are available in experimental + build, namely round-robin. sectorsize (RO, since: 3.14) Shows the sectorsize of the mounted filesystem. - temp_fsid (RO, since 6.7) Indicate that this filesystem got assigned a temporary FSID at mount time, making possible to mount devices with the same FSID. +UUID/allocations +^^^^^^^^^^^^^^^^ + Files and directories in :file:`/sys/fs/btrfs//allocations` directory are: global_rsv_reserved @@ -136,8 +145,10 @@ global_rsv_size `data/`, `metadata/` and `system/` directories (RO, since: 5.14) - Space info accounting for the 3 chunk types. - Mostly for debug purposes. + Space info accounting for the 3 block group types. + +UUID/allocations/{data,metadata,system} +""""""""""""""""""""""""""""""""""""""" Files in :file:`/sys/fs/btrfs//allocations/{data,metadata,system}` directory are: @@ -152,13 +163,20 @@ bytes_* (RO) Values of the corresponding data structures for the given block group - type and profile. + type and profile that are used internally and may change rapidly depending + on the load. + + Complete list: bytes_may_use, bytes_pinned, bytes_readonly, + bytes_reserved, bytes_used, bytes_zone_unusable chunk_size (RW, since: 6.0) - Shows the chunk size. Can be changed for data and metadata. - Cannot be set for zoned devices. + Shows the chunk size. Can be changed for data and metadata (independently) + and cannot be set for system block group type. + Cannot be set for zoned devices as it depends on the fixed device zone size. + Upper bound is 10% of the filesystem size, the value must be multiple of 256MiB + and greater than 0. size_classes (RO, since: 6.3) @@ -173,6 +191,27 @@ size_classes medium 282 large 93 +UUID/bdi +^^^^^^^^ + +Symlink to the sysfs directory of the backing device info (BDI), which is +related to writeback process and infrastructure. + +UUID/devices +^^^^^^^^^^^^ + +Files in :file:`/sys/fs/btrfs//devices` directory are symlinks named +after device nodes (e.g. sda, dm-0) and pointing to their sysfs directory. + +UUID/devinfo +^^^^^^^^^^^^ + +The directory contains subdirectories named after device ids (numeric values). Each +subdirectory has information about the device of the given *devid*. + +UUID/devinfo/DEVID +"""""""""""""""""" + Files in :file:`/sys/fs/btrfs//devinfo/` directory are: error_stats: @@ -192,25 +231,25 @@ fsid: (RO, since: 5.17) Shows the fsid which the device belongs to. - It can be different than the `` if it's a seed device. + It can be different than the ``UUID`` if it's a seed device. in_fs_metadata (RO, since: 5.6) Shows whether we have found the device. - Should always be 1, as if this turns to 0, the `` directory + Should always be 1, as if this turns to 0, the :file:`DEVID` directory would get removed automatically. missing (RO, since: 5.6) - Shows whether the device is missing. + Shows whether the device is considered missing by the kernel module. replace_target (RO, since: 5.6) Shows whether the device is the replace target. - If no dev-replace is running, this value should be 0. + If no device replace is running, this value is 0. scrub_speed_max (RW, since: 5.14) @@ -223,13 +262,16 @@ writeable Show if the device is writeable. +UUID/qgroups +^^^^^^^^^^^^ + Files in :file:`/sys/fs/btrfs//qgroups/` directory are: enabled (RO, since: 6.1) Shows if qgroup is enabled. - Also, if qgroup is disabled, the `qgroups` directory would + Also, if qgroup is disabled, the :file:`qgroups` directory will be removed automatically. inconsistent @@ -249,12 +291,17 @@ drop_subtree_threshold we will not trigger qgroup account at all, but mark qgroup inconsistent to avoid the huge workload. - Default value is 8, where no subtree drop can trigger qgroup. + Default value is 3, which means that trees of low height will be accounted + properly as this is sufficiently fast. The value was 8 until 6.13 where + no subtree drop can trigger qgroup rescan making it less useful. Lower value can reduce qgroup workload, at the cost of extra qgroup rescan to re-calculate the numbers. -Files in :file:`/sys/fs/btrfs//_/` directory are: +UUID/qgroups/LEVEL_ID +""""""""""""""""""""" + +Files in each :file:`/sys/fs/btrfs//qgroups/_/` directory are: exclusive (RO, since: 5.9) @@ -297,6 +344,9 @@ rsv_meta_prealloc Shows the reserved bytes for preallocated metadata. +UUID/discard +^^^^^^^^^^^^ + Files in :file:`/sys/fs/btrfs//discard/` directory are: discardable_bytes From 49faefa4894b77dabbe5dcf4d0ce74b33d4060d1 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 12 Feb 2025 03:29:51 +0100 Subject: [PATCH 14/27] btrfs-progs: docs: fix document references in html build The references used in :doc: need to reference relative path of the document, otherwise this leads to a warning. btrfs-progs/Documentation/Send-receive.rst:19: WARNING: unknown document: 'dev-send-stream' [ref.doc] btrfs-progs/Documentation/dev/CmdLineConventions.rst:60: WARNING: unknown document: 'btrfstune' [ref.doc] Signed-off-by: David Sterba --- Documentation/Send-receive.rst | 2 +- Documentation/dev/CmdLineConventions.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/Send-receive.rst b/Documentation/Send-receive.rst index c0e8dee56..d0a75e6a7 100644 --- a/Documentation/Send-receive.rst +++ b/Documentation/Send-receive.rst @@ -22,4 +22,4 @@ truncate), whole file operations (rename, delete). The stream can be sent over network, piped directly to the receive command or saved to a file. Each command in the stream is protected by a CRC32C checksum, with 0 as the initial value and no inversion. See :doc:`btrfs-send` and :doc:`btrfs-receive` for more, -for protocol description :doc:`dev-send-stream`. +for protocol description :doc:`dev/dev-send-stream`. diff --git a/Documentation/dev/CmdLineConventions.rst b/Documentation/dev/CmdLineConventions.rst index 72ac875ca..1c3034b20 100644 --- a/Documentation/dev/CmdLineConventions.rst +++ b/Documentation/dev/CmdLineConventions.rst @@ -58,7 +58,7 @@ or change descriptions. * brief but descriptive * long and descriptive can be used in justified cases (e.g. conversion options - in :doc:`btrfstune` because of the single command without subcommands) + in :doc:`../btrfstune` because of the single command without subcommands) Option parameters ^^^^^^^^^^^^^^^^^ From 7535901b06e525996585a5a0ad44927b2b2e1a7d Mon Sep 17 00:00:00 2001 From: Mark Harmstone Date: Thu, 6 Feb 2025 11:24:46 +0000 Subject: [PATCH 15/27] btrfs-progs: mkfs: set BTRFS_INODE_ROOT_ITEM_INIT on new FS root The kernel commit 08fe4db170b419 ("Btrfs: Fix uninitialized root flags for subvolumes") from 2011 sets the flag BTRFS_INODE_ROOT_ITEM_INIT on root items, to work around a bug where flags and byte_limit weren't being set. Copy this behaviour in mkfs, to prevent the kernel from having to do it on the first mount. We memset the btrfs_root_item, so there's no corruption issue as there once was. We already do this in btrfs_make_subvolume(), as otherwise the readonly flag of any subvolumes would get reset. Signed-off-by: Mark Harmstone Signed-off-by: David Sterba --- mkfs/common.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mkfs/common.c b/mkfs/common.c index f6501a130..249e6f3da 100644 --- a/mkfs/common.c +++ b/mkfs/common.c @@ -127,11 +127,13 @@ static int btrfs_create_tree_root(int fd, struct btrfs_mkfs_config *cfg, memcpy(root_item.uuid, uuid, BTRFS_UUID_SIZE); btrfs_set_stack_timespec_sec(&root_item.otime, now); btrfs_set_stack_timespec_sec(&root_item.ctime, now); + btrfs_set_stack_inode_flags(inode_item, BTRFS_INODE_ROOT_ITEM_INIT); } else { memset(uuid, 0, BTRFS_UUID_SIZE); memcpy(root_item.uuid, uuid, BTRFS_UUID_SIZE); btrfs_set_stack_timespec_sec(&root_item.otime, 0); btrfs_set_stack_timespec_sec(&root_item.ctime, 0); + btrfs_set_stack_inode_flags(inode_item, 0); } write_extent_buffer(buf, &root_item, btrfs_item_ptr_offset(buf, nritems), From 80cd98ee5917e943e8962754f913d0dd095ef852 Mon Sep 17 00:00:00 2001 From: Mark Harmstone Date: Thu, 6 Feb 2025 14:32:11 +0000 Subject: [PATCH 16/27] btrfs-progs: mkfs: add dev stats and device items The kernel adds a zeroed btrfs_dev_stats_item for each device on the first mount. Preempt this by doing it at mkfs time. Signed-off-by: Mark Harmstone Signed-off-by: David Sterba --- kernel-shared/ctree.h | 10 ++++++++++ kernel-shared/volumes.c | 23 +++++++++++++++++++++++ mkfs/common.c | 20 +++++++++++++++++++- 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/kernel-shared/ctree.h b/kernel-shared/ctree.h index 6ba982ef5..8c923be96 100644 --- a/kernel-shared/ctree.h +++ b/kernel-shared/ctree.h @@ -831,6 +831,16 @@ static inline u64 btrfs_dev_stats_value(const struct extent_buffer *eb, return val; } +static inline void btrfs_set_dev_stats_value(struct extent_buffer *eb, + struct btrfs_dev_stats_item *ptr, + int index, u64 val) +{ + write_extent_buffer(eb, &val, + offsetof(struct btrfs_dev_stats_item, values) + + ((unsigned long)ptr) + (index * sizeof(u64)), + sizeof(val)); +} + /* struct btrfs_ioctl_search_header */ static inline u64 btrfs_search_header_transid(struct btrfs_ioctl_search_header *sh) { diff --git a/kernel-shared/volumes.c b/kernel-shared/volumes.c index b21231efe..a0842d58d 100644 --- a/kernel-shared/volumes.c +++ b/kernel-shared/volumes.c @@ -1135,6 +1135,7 @@ int btrfs_add_device(struct btrfs_trans_handle *trans, int ret; struct btrfs_path *path; struct btrfs_dev_item *dev_item; + struct btrfs_dev_stats_item *dev_stats; struct extent_buffer *leaf; struct btrfs_key key; struct btrfs_root *root = fs_info->chunk_root; @@ -1149,6 +1150,7 @@ int btrfs_add_device(struct btrfs_trans_handle *trans, if (ret) goto out; + /* Add DEV_ITEM. */ key.objectid = BTRFS_DEV_ITEMS_OBJECTID; key.type = BTRFS_DEV_ITEM_KEY; key.offset = free_devid; @@ -1182,6 +1184,27 @@ int btrfs_add_device(struct btrfs_trans_handle *trans, BTRFS_UUID_SIZE); btrfs_mark_buffer_dirty(leaf); fs_info->fs_devices->total_rw_bytes += device->total_bytes; + + btrfs_release_path(path); + + /* Add DEV STATS item. */ + key.objectid = BTRFS_DEV_STATS_OBJECTID; + key.type = BTRFS_PERSISTENT_ITEM_KEY; + key.offset = free_devid; + + ret = btrfs_insert_empty_item(trans, fs_info->dev_root, path, &key, + sizeof(*dev_stats)); + if (ret) + goto out; + + leaf = path->nodes[0]; + dev_stats = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_stats_item); + + for (int i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) + btrfs_set_dev_stats_value(leaf, dev_stats, i, 0); + + btrfs_mark_buffer_dirty(leaf); + ret = 0; out: diff --git a/mkfs/common.c b/mkfs/common.c index 249e6f3da..bb5a2ad46 100644 --- a/mkfs/common.c +++ b/mkfs/common.c @@ -369,6 +369,7 @@ int make_btrfs(int fd, struct btrfs_mkfs_config *cfg) struct btrfs_chunk *chunk; struct btrfs_dev_item *dev_item; struct btrfs_dev_extent *dev_extent; + struct btrfs_dev_stats_item *dev_stats; enum btrfs_mkfs_block blocks[MKFS_BLOCK_COUNT]; u8 chunk_tree_uuid[BTRFS_UUID_SIZE]; u8 *ptr; @@ -673,8 +674,25 @@ int make_btrfs(int fd, struct btrfs_mkfs_config *cfg) memset(buf->data + sizeof(struct btrfs_header), 0, cfg->nodesize - sizeof(struct btrfs_header)); nritems = 0; - itemoff = cfg->leaf_data_size - sizeof(struct btrfs_dev_extent); + itemoff = cfg->leaf_data_size; + + /* Add a DEV_STATS item for device 1. */ + itemoff -= sizeof(struct btrfs_dev_stats_item); + btrfs_set_disk_key_objectid(&disk_key, BTRFS_DEV_STATS_OBJECTID); + btrfs_set_disk_key_type(&disk_key, BTRFS_PERSISTENT_ITEM_KEY); + btrfs_set_disk_key_offset(&disk_key, 1); + btrfs_set_item_key(buf, &disk_key, nritems); + btrfs_set_item_offset(buf, nritems, itemoff); + btrfs_set_item_size(buf, nritems, sizeof(struct btrfs_dev_stats_item)); + dev_stats = btrfs_item_ptr(buf, nritems, struct btrfs_dev_stats_item); + + for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) + btrfs_set_dev_stats_value(buf, dev_stats, i, 0); + + nritems++; + /* Add the DEV_EXTENT item for the system chunk. */ + itemoff -= sizeof(struct btrfs_dev_extent); btrfs_set_disk_key_objectid(&disk_key, 1); btrfs_set_disk_key_offset(&disk_key, system_group_offset); btrfs_set_disk_key_type(&disk_key, BTRFS_DEV_EXTENT_KEY); From 2bb184dcd50d598a8f577c5f705ae5487d798a59 Mon Sep 17 00:00:00 2001 From: Mark Harmstone Date: Thu, 6 Feb 2025 14:41:53 +0000 Subject: [PATCH 17/27] btrfs-progs: mkfs: set incompat bit BIG_METADATA if nodesize > PAGE_SIZE Follow the kernel by setting the BIG_METADATA incompat flag if nodesize is greater than the page size. This flag was introduced with commit 727011e07cbdf8 ("Btrfs: allow metadata blocks larger than the page size") in 2010, as kernels before 2.6.36 would crash due to a buggy page cache implementation. The flag has no real meaning anymore but we can at least set it at mkfs time. Signed-off-by: Mark Harmstone Signed-off-by: David Sterba --- mkfs/main.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mkfs/main.c b/mkfs/main.c index 3e5e18c3b..92441710f 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -1665,6 +1665,10 @@ int BOX_MAIN(mkfs)(int argc, char **argv) goto error; } + /* This is also fixed in kernel, but the flag has no real meaning anymore. */ + if (nodesize > sysconf(_SC_PAGE_SIZE)) + features.incompat_flags |= BTRFS_FEATURE_INCOMPAT_BIG_METADATA; + if (sectorsize < sizeof(struct btrfs_super_block)) { error("sectorsize smaller than superblock: %u < %zu", sectorsize, sizeof(struct btrfs_super_block)); From e15bba1f45a044460c69f95d4d45809161784734 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 16 Jan 2025 09:01:21 +1030 Subject: [PATCH 18/27] btrfs-progs: docs: add a note on qgroup limit with inconsitent flag Just like all qgroup functions, if a qgroup is marked inconsistent, limit will not work as expected. In fact with recent kernels, limit and qgroup number updating will be fully skipped if qgroup is already inconsistent. Add one extra note on `btrfs qgroup limit` subcommand for it. Link: https://bugzilla.suse.com/show_bug.cgi?id=1235765 Reported-by: Vojtech Lacina Signed-off-by: Qu Wenruo --- Documentation/btrfs-qgroup.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Documentation/btrfs-qgroup.rst b/Documentation/btrfs-qgroup.rst index b90cfbb54..0c5a00988 100644 --- a/Documentation/btrfs-qgroup.rst +++ b/Documentation/btrfs-qgroup.rst @@ -31,6 +31,12 @@ ownership. For example a fresh snapshot shares almost all the blocks with the original subvolume, new writes to either subvolume will raise towards the exclusive limit. +.. note:: + Qgroup limit only works when qgroup is in a consistent state. + If some workload marks qgroup inconsistent (like assigning a qgroup to another + qgroup), the limit will no longer work until the inconsistent flag is cleared + by :command:`btrfs quota rescan`. + The qgroup identifiers conform to *level/id* where level 0 is reserved to the qgroups associated with subvolumes. Such qgroups are created automatically. From 5a24fb191d66783a6375c8d6ff95c70db1a029bb Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Mon, 3 Feb 2025 13:12:37 +1030 Subject: [PATCH 19/27] btrfs-progs: docs: add a warning when converting to a profile with lower redundancy [BUG] There is a bug report that when deleting a device using sysfs /sys/block//device/delete, the kernel module will still try to read and write the device. Normally it's fine as long as all chunks can tolerate that removed device (e.g. all RAID1). But the problem is when one is trying to lower the redundancy by converting to another profile: # mkfs.btrfs -f -m raid1 -d raid1 /dev/sdd /dev/sde # mount /dev/sdd /mnt # echo 1 > /sys/block/sde/device/delete # btrfs balance start --force -mdup -dsingle /mnt This will lead to the filesystem mounted RO, with the following error messages: sd 6:0:0:0: [sde] Synchronizing SCSI cache ata7.00: Entering standby power mode btrfs: attempt to access beyond end of device sde: rw=6145, sector=21696, nr_sectors = 32 limit=0 btrfs: attempt to access beyond end of device sde: rw=6145, sector=21728, nr_sectors = 32 limit=0 btrfs: attempt to access beyond end of device sde: rw=6145, sector=21760, nr_sectors = 32 limit=0 BTRFS error (device sdd): bdev /dev/sde errs: wr 1, rd 0, flush 0, corrupt 0, gen 0 BTRFS error (device sdd): bdev /dev/sde errs: wr 2, rd 0, flush 0, corrupt 0, gen 0 BTRFS error (device sdd): bdev /dev/sde errs: wr 3, rd 0, flush 0, corrupt 0, gen 0 BTRFS error (device sdd): bdev /dev/sde errs: wr 3, rd 0, flush 1, corrupt 0, gen 0 btrfs: attempt to access beyond end of device sde: rw=145409, sector=128, nr_sectors = 8 limit=0 BTRFS warning (device sdd): lost super block write due to IO error on /dev/sde (-5) BTRFS error (device sdd): bdev /dev/sde errs: wr 4, rd 0, flush 1, corrupt 0, gen 0 btrfs: attempt to access beyond end of device sde: rw=14337, sector=131072, nr_sectors = 8 limit=0 BTRFS warning (device sdd): lost super block write due to IO error on /dev/sde (-5) BTRFS error (device sdd): bdev /dev/sde errs: wr 5, rd 0, flush 1, corrupt 0, gen 0 BTRFS error (device sdd): error writing primary super block to device 2 BTRFS info (device sdd): balance: start -dconvert=single -mconvert=dup -sconvert=dup BTRFS info (device sdd): relocating block group 1372585984 flags data|raid1 BTRFS error (device sdd): bdev /dev/sde errs: wr 5, rd 0, flush 2, corrupt 0, gen 0 BTRFS warning (device sdd): chunk 2446327808 missing 1 devices, max tolerance is 0 for writable mount BTRFS: error (device sdd) in write_all_supers:4044: errno=-5 IO failure (errors while submitting device barriers.) BTRFS info (device sdd state E): forced readonly BTRFS warning (device sdd state E): Skipping commit of aborted transaction. BTRFS error (device sdd state EA): Transaction aborted (error -5) BTRFS: error (device sdd state EA) in cleanup_transaction:2017: errno=-5 IO failure BTRFS info (device sdd state EA): balance: ended with status: -5 [CAUSE] Btrfs doesn't have any runtime device error handling, it fully rely on the extra copy provided. For the sysfs block device removal, normally there is a device shutdown callback to the running fs, but unfortunately btrfs doesn't support this callback yet. Thus even with that device removed, btrfs will still access that removed device (both read and write, even if they will fail). Normally for a full RAID1 btrfs, it will still be fine reading/write the fs as usual. The proper action is to replace the removed/missing/failing device with a newer one using `btrfs device replace`. But when doing the convert, btrfs will allocate new metadata chunks on to the removed device (which will lose all writes). And since the new metadata profile is DUP, which can not handle any missing device of that metadata chunk, finally it triggers the final protection at transaction commit time, and flips the filesystem to RO, before causing any real data loss. [DOC ENHANCEMENT] Add a warning to the `convert` filter about the dangerous doing convert to a lower redundancy profile when there is a known failing/removed device. And mention the proper way to handle such failing/missing device. The root fix is to introduce a failing/removed device detection for btrfs, but that will be a pretty big feature and will take quite some time before landing it upstream. Link: https://lore.kernel.org/linux-btrfs/2cb1d81e-12a8-4fb1-b3fc-e7e83d31e059@siddall.name/ Reported-by: Jeff Siddall Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- Documentation/ch-balance-filters.rst | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/Documentation/ch-balance-filters.rst b/Documentation/ch-balance-filters.rst index 81a0ca8f0..46b8fb92c 100644 --- a/Documentation/ch-balance-filters.rst +++ b/Documentation/ch-balance-filters.rst @@ -79,10 +79,26 @@ convert= Starting with kernel 4.5, the ``data`` chunks can be converted to/from the ``DUP`` profile on a single device. - .. note:: Starting with kernel 4.6, all profiles can be converted to/from ``DUP`` on multi-device filesystems. + .. warning:: + Bad or missing device are not detected immediately during + runtime and this depends on some later event like failed write + or failed transaction commit. If there's a known failing + device, or a device deleted by :file:`/sys/block//device/delete` interface, + the device will be still accessed and written to. + + In such case, one should not convert to a profile with lower + redundancy (e.g. from *RAID1* to *SINGLE*), + as attempts to create new chunks on the new devices will cause + various problems. + + The proper action is to use :command:`btrfs replace` or + :command:`btrfs device remove` to handle the failing/missing + device first. Then convert will work with all devices + correctly. + limit=, limit= Process only given number of chunks, after all filters are applied. This can be used to specifically target a chunk in connection with other filters (``drange``, From e1085e6287b11dd68a21ab0cbb72a841e9f27fce Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 12 Feb 2025 22:55:22 +0100 Subject: [PATCH 20/27] btrfs-progs: tests: adjust number of misc/065-image-filename There already is misc/065-csum-conversion-inject . Signed-off-by: David Sterba --- .../misc-tests/{065-image-filename => 066-image-filename}/test.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/misc-tests/{065-image-filename => 066-image-filename}/test.sh (100%) diff --git a/tests/misc-tests/065-image-filename/test.sh b/tests/misc-tests/066-image-filename/test.sh similarity index 100% rename from tests/misc-tests/065-image-filename/test.sh rename to tests/misc-tests/066-image-filename/test.sh From 92ab5b553fd6054f36316ad361d9240f153f052a Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 12 Feb 2025 22:55:22 +0100 Subject: [PATCH 21/27] btrfs-progs: tests: adjust number of misc/065-btrfstune-simple-quota The 065 and 066 are taken. Signed-off-by: David Sterba --- .../test.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/misc-tests/{065-btrfstune-simple-quota => 067-btrfstune-simple-quota}/test.sh (100%) diff --git a/tests/misc-tests/065-btrfstune-simple-quota/test.sh b/tests/misc-tests/067-btrfstune-simple-quota/test.sh similarity index 100% rename from tests/misc-tests/065-btrfstune-simple-quota/test.sh rename to tests/misc-tests/067-btrfstune-simple-quota/test.sh From 792fe152db0228208cb5377ee7a2a6147d3b75be Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 12 Feb 2025 22:58:37 +0100 Subject: [PATCH 22/27] btrfs-progs: tests: adjust number of misc/064-reverse-receive There are two tests 064, and we're missing 061 so fill the gap. Signed-off-by: David Sterba --- .../{064-reverse-receive => 061-reverse-receive}/test.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/misc-tests/{064-reverse-receive => 061-reverse-receive}/test.sh (100%) diff --git a/tests/misc-tests/064-reverse-receive/test.sh b/tests/misc-tests/061-reverse-receive/test.sh similarity index 100% rename from tests/misc-tests/064-reverse-receive/test.sh rename to tests/misc-tests/061-reverse-receive/test.sh From ab0d5e2149621df0b30647b7da627c9e839a7743 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 12 Feb 2025 23:01:19 +0100 Subject: [PATCH 23/27] btrfs-progs: tests: rename and update convert/027-large-symbol-link Use 'symbolic' as the term. Signed-off-by: David Sterba --- .../test.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) rename tests/convert-tests/{027-large-symbol-link => 027-large-symbolic-link}/test.sh (60%) diff --git a/tests/convert-tests/027-large-symbol-link/test.sh b/tests/convert-tests/027-large-symbolic-link/test.sh similarity index 60% rename from tests/convert-tests/027-large-symbol-link/test.sh rename to tests/convert-tests/027-large-symbolic-link/test.sh index 2a001424d..e88f2939a 100755 --- a/tests/convert-tests/027-large-symbol-link/test.sh +++ b/tests/convert-tests/027-large-symbolic-link/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Make sure btrfs-convert can handle a symbol link which is 4095 bytes large +# Make sure btrfs-convert can handle a symbolic link which is 4095 bytes long source "$TEST_TOP/common" || exit source "$TEST_TOP/common.convert" || exit @@ -12,16 +12,16 @@ check_global_prereq mkfs.ext4 link_target=$(printf "%0.sb" {1..4095}) convert_test_prep_fs ext4 mke2fs -t ext4 -b 4096 -run_check $SUDO_HELPER ln -s "$link_target" "$TEST_MNT/symbol_link" +run_check $SUDO_HELPER ln -s "$link_target" "$TEST_MNT/symbolic_link" run_check_umount_test_dev # For unpatched btrfs-convert, it will always append one byte to the -# link target, causing above 4095 target to be 4096, exactly one sector, -# resulting a regular file extent. +# link target, causing the above 4095 target to be 4096, exactly one sector, +# resulting in a regular file extent. convert_test_do_convert run_check_mount_test_dev # If the unpatched btrfs-convert created a regular extent, and the kernel is -# newer enough, such readlink will be rejected by kernel. -run_check $SUDO_HELPER readlink "$TEST_MNT/symbol_link" +# new enough, readlink will be rejected by kernel. +run_check $SUDO_HELPER readlink "$TEST_MNT/symbolic_link" run_check_umount_test_dev From f6ec4de69785c710f0b45d83ebdde3389f0b94de Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 12 Feb 2025 23:23:53 +0100 Subject: [PATCH 24/27] btrfs-progs: tests: add prefix to test templates and update permissions The templates need to be renamed, so make it more suggestive by prepending the expected name of the script test.sh. Also fix the permissions and make it 755 so it's not missed later as this would not execute the script. Signed-off-by: David Sterba --- tests/template/{check-all-images => test.sh.check-all-images} | 0 tests/template/{multi-mkfs-mount => test.sh.multi-mkfs-mount} | 0 tests/template/{single-mkfs-mount => test.sh.single-mkfs-mount} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename tests/template/{check-all-images => test.sh.check-all-images} (100%) mode change 100644 => 100755 rename tests/template/{multi-mkfs-mount => test.sh.multi-mkfs-mount} (100%) mode change 100644 => 100755 rename tests/template/{single-mkfs-mount => test.sh.single-mkfs-mount} (100%) mode change 100644 => 100755 diff --git a/tests/template/check-all-images b/tests/template/test.sh.check-all-images old mode 100644 new mode 100755 similarity index 100% rename from tests/template/check-all-images rename to tests/template/test.sh.check-all-images diff --git a/tests/template/multi-mkfs-mount b/tests/template/test.sh.multi-mkfs-mount old mode 100644 new mode 100755 similarity index 100% rename from tests/template/multi-mkfs-mount rename to tests/template/test.sh.multi-mkfs-mount diff --git a/tests/template/single-mkfs-mount b/tests/template/test.sh.single-mkfs-mount old mode 100644 new mode 100755 similarity index 100% rename from tests/template/single-mkfs-mount rename to tests/template/test.sh.single-mkfs-mount From d12e76e7f5a801a66f6585d093a5978814219417 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 12 Feb 2025 23:25:18 +0100 Subject: [PATCH 25/27] btrfs-progs: tests: update README Minor fixes and updates that reflect current state. [ci skip] Signed-off-by: David Sterba --- tests/README.md | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/tests/README.md b/tests/README.md index 8e5d59a5f..1bd0e7c3f 100644 --- a/tests/README.md +++ b/tests/README.md @@ -8,6 +8,12 @@ management. In some cases optional features are turned on by mkfs and the filesystem image could be mounted, such tests might fail if there's lack of support. +The testsuite follows the principles: + +- gather all output to logs +- stop on first failure (do not clean up until the problem is investigated) +- detect available capabilities in the system, skip if missing + ## Quick start Run the tests from the top directory: @@ -199,7 +205,7 @@ command `losetup` and eventually delete all existing loop devices with `losetup ### Prototyping tests, quick tests There's a script `test-console.sh` that will run shell commands in a loop and -logs the output with the testing environment set up. It sources the common +log the output with the testing environment set up. It sources the common helper scripts so the shell functions are available. ### Runtime dependencies @@ -213,7 +219,8 @@ specific tests need the following packages installed: `acl`, `attr`, 1. Pick the category for the new test or fallback to `misc-tests` if not sure. For an easy start copy an existing `test.sh` script from some test that might be -close to the purpose of your new test. The environment setup includes the +close to the purpose of your new test, or use one of the templates in the +[tests/template/](template/) directory. The environment setup includes the common scripts and/or prepares the test devices. Other scripts contain examples how to do mkfs, mount, unmount, check, loop device management etc. @@ -229,22 +236,23 @@ infrastructure. 5. **Test your test.** Use the `TEST` variable to jump right to your test: ```shell -$ make TEST=012\* tests-misc # from top directory -$ TEST=012\* ./misc-tests.sh # from tests/ +$ make TEST=012\* tests-misc # from the top or tests/ directory +$ TEST=012\* ./misc-tests.sh # alternatively from tests/ ``` 6. The commit changelog should reference a commit that either introduced or - fixed the bug (or both). Subject line of the shall mention the name of the - new directory for ease of search, e.g. `btrfs-progs: tests: add 012-subvolume-sync-must-wait` + fixed the bug (or both). Subject line of the could mention the name of the + new directory for ease of search (e.g. `btrfs-progs: tests: add 012-subvolume-sync-must-wait`) + or a brief description of the test case (e.g. `btrfs-progs: tests: add case for waiting on subvolume sync after deletion`) -7. A commit that fixes a bug should be applied before the test that verifies +7. A commit that fixes a bug should be applied *before* the test that verifies the fix. This is to keep the git history bisectable. ### Test images Most tests should be able to create the test images from scratch, using regular -commands and file operation. The commands also document the test case and use +commands and file operations. The commands also document the test case and use the test code and kernel of the environment. In other cases, a pre-created image may be the right way if the above does not @@ -295,7 +303,8 @@ There are some utilities that are not distributed but are necessary for the tests. They are in the top level directory of the testsuite and their path cannot be set. -The tests assume write access to their directories. +The tests assume write access to their directories and an existing `/tmp` +directory. # Coding style, best practices @@ -305,9 +314,10 @@ The tests assume write access to their directories. * quote all variables by default, any path, even the TOP could need that, and we use it everywhere * even if the variable is safe, use quotes for consistency and to ease - reading the code + reading the code (syntax highlighting in editors) * there are exceptions: * `$SUDO_HELPER` as it might be intentionally unset + * conditionally set options that are stored in a variable * use `#!/bin/bash` explicitly * check for all external dependencies (`check_global_prereq`) * check for internal dependencies (`check_prereq`), though the basic set is From a1926e39df13813ea4130f656a72b6052072848e Mon Sep 17 00:00:00 2001 From: David Sterba Date: Thu, 13 Feb 2025 00:00:08 +0100 Subject: [PATCH 26/27] btrfs-progs: ci: add option to run only codespell workflow For quick checks before a push of non-code changes we may want to do only the spellchecking workflow. Any branch pushed matching the prefix "codespell/" will be picked by this. Signed-off-by: David Sterba --- .github/workflows/codespell.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index c9f92e2f7..f9a29030e 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -1,5 +1,7 @@ # Check for typos on devel and pull rquests # - codespell configuration is within .codespellrc +# - the workflow can be run separately from other workflows that trigger on +# ci/* branches, use codespell/* as the prefix name: Codespell run-name: Codespell @@ -9,12 +11,11 @@ on: branches: - devel - 'ci/**' - - 'CI/**' + - 'codespell/**' pull_request: branches: - devel - 'ci/**' - - 'CI/**' permissions: contents: read From efd8a2e2a9d8e19c699815a3e3506f6366995495 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 13 Feb 2025 14:52:35 +1030 Subject: [PATCH 27/27] btrfs-progs: cmds/qgroup: use sysfs to detect inconsistent status early Currently if "btrfs qgroup show" detects the qgroup is in an inconsistent, it will output a warning: WARNING: qgroup data inconsistent, rescan recommended But the detection is based on the tree search ioctl, and qgroup tree is only updated at transaction commit time. This means if qgroup is marked inconsistent, and the transaction is not commit, there can be a huge window as long as 30s before "btrfs qgroup show" to give a proper warning about inconsistent qgroup numbers. To address this window, use the '/sys/fs/btrfs//qgroup/inconsistent' file to do extra check. That file is updated at real time, thus there is no delay, and can give an early warning about inconsistent qgroup numbers. Link: https://bugzilla.suse.com/show_bug.cgi?id=1235765 Signed-off-by: Qu Wenruo --- cmds/qgroup.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/cmds/qgroup.c b/cmds/qgroup.c index a612130c9..45439953b 100644 --- a/cmds/qgroup.c +++ b/cmds/qgroup.c @@ -36,6 +36,7 @@ #include "kernel-shared/uapi/btrfs.h" #include "kernel-shared/ctree.h" #include "common/open-utils.h" +#include "common/sysfs-utils.h" #include "common/utils.h" #include "common/help.h" #include "common/units.h" @@ -1610,6 +1611,33 @@ static void print_all_qgroups_json(struct qgroup_lookup *qgroup_lookup) fmt_end(&fctx); } +/* + * Tree search based inconsistent flag is only updated at transaction commit + * time. + * Thus even if the qgroup_status flag shows consistent, the qgroup may already + * be in an inconsistent status. + */ +static void check_qgroup_sysfs_inconsistent(int fd, + const struct qgroup_lookup *qgroup_lookup) +{ + u64 value; + int sysfs_fd; + int ret; + + if (qgroup_lookup->flags & BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT) + return; + sysfs_fd = sysfs_open_fsid_file(fd, "qgroups/inconsistent"); + if (fd < 0) + return; + ret = sysfs_read_fsid_file_u64(fd, "qgroups/inconsistent", &value); + if (ret < 0) + goto out; + if (value) + warning("qgroup data inconsistent, rescan recommended"); +out: + close(sysfs_fd); +} + static int show_qgroups(int fd, struct btrfs_qgroup_filter_set *filter_set, struct btrfs_qgroup_comparer_set *comp_set) @@ -1622,6 +1650,8 @@ static int show_qgroups(int fd, ret = qgroups_search_all(fd, &qgroup_lookup); if (ret) return ret; + + check_qgroup_sysfs_inconsistent(fd, &qgroup_lookup); __filter_and_sort_qgroups(&qgroup_lookup, &sort_tree, filter_set, comp_set); if (bconf.output_format == CMD_FORMAT_JSON)