diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000000..e2700d27a1 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,105 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL Advanced" + +on: + push: + branches: [ "devel" ] + pull_request: + branches: [ "devel" ] + schedule: + - cron: '37 22 * * 0' + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners (GitHub.com only) + # Consider using larger runners or machines with greater resources for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + permissions: + # required for all workflows + security-events: write + + # required to fetch internal or private CodeQL packs + packages: read + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + include: + - language: c-cpp + build-mode: manual + - language: python + build-mode: none + # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' + # Use `c-cpp` to analyze code written in C, C++ or both + # Use 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, + # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. + # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how + # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Add any setup steps before running the `github/codeql-action/init` action. + # This includes steps like installing compilers or runtimes (`actions/setup-node` + # or others). This is typically only required for manual builds. + # - name: Setup runtime (example) + # uses: actions/setup-example@v1 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + - name: Update package cache + run: sudo apt-get update -qqq + - name: Add more build dependencies + run: sudo apt-get install -y liblzo2-dev libzstd-dev libblkid-dev uuid-dev zlib1g-dev libext2fs-dev e2fsprogs libudev-dev + + # If the analyze step fails for one of the languages you are analyzing with + # "We were unable to automatically build your code", modify the matrix above + # to set the build mode to "manual" for that language. Then modify this step + # to build your code. + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + - if: matrix.build-mode == 'manual' + shell: bash + run: | + echo 'If you are using a "manual" build mode for one or more of the' \ + 'languages you are analyzing, replace this with the commands to build' \ + 'your code, for example:' + ./autogen.sh + ./configure --disable-documentation --enable-experimental + make -j 4 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index c9f92e2f73..f9a29030e0 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -1,5 +1,7 @@ # Check for typos on devel and pull rquests # - codespell configuration is within .codespellrc +# - the workflow can be run separately from other workflows that trigger on +# ci/* branches, use codespell/* as the prefix name: Codespell run-name: Codespell @@ -9,12 +11,11 @@ on: branches: - devel - 'ci/**' - - 'CI/**' + - 'codespell/**' pull_request: branches: - devel - 'ci/**' - - 'CI/**' permissions: contents: read diff --git a/Documentation/DocConventions.rst b/Documentation/DocConventions.rst index 96a96c6172..955b12e574 100644 --- a/Documentation/DocConventions.rst +++ b/Documentation/DocConventions.rst @@ -67,7 +67,7 @@ Referencing: - generic links can use the free form link syntax with description ```Link text `__`` (note the double underscore, this is *anonymous* link and does not create a reference) - or plain link that will auto-render to a clikable link https://example.com + or plain link that will auto-render to a clickable link https://example.com - in manual pages: always use full link as it's meant to be read in terminal output and must allow copy&paste diff --git a/Documentation/Kernel-by-version.rst b/Documentation/Kernel-by-version.rst index 980946c5da..f2dd36d18e 100644 --- a/Documentation/Kernel-by-version.rst +++ b/Documentation/Kernel-by-version.rst @@ -853,7 +853,7 @@ Pull requests: `v5.4-rc7 `__, `v5.4-rc8 `__ -- tree checker: adde sanity checks for tree items, extent items, and references +- tree checker: added sanity checks for tree items, extent items, and references - deprecated subvolume creation mode BTRFS_SUBVOL_CREATE_ASYNC - qgroup relation deletion tries harder, orphan entries are removed too - space handling improvements (ticket reservations, flushing, overcommit logic) @@ -1023,7 +1023,7 @@ Pull requests: Highlights: - performance improvements in fsync (dbench workload: higher throughput, lower latency) -- sysfs exports current exclusive operataion (balance, resize, device add/del/...) +- sysfs exports current exclusive operation (balance, resize, device add/del/...) - sysfs exports supported send stream version Core: @@ -1083,7 +1083,7 @@ Features: Core changes: -- subpage block size support peparations +- subpage block size support preparations Fixes: @@ -1434,7 +1434,7 @@ Qgroup: - limits are shared upon snapshot - allow to remove qgroup which has parent but no child - fix status of qgroup consistency after rescan -- fix quota status bits after dsiabling +- fix quota status bits after disabling - mark qgroups inconsistent after assign/delete actions - code cleanups @@ -1475,7 +1475,7 @@ Fixes: ^^^^^^^^^^^^^^ - send fixes: cloning, sending with parent -- improved handling of framgented space using bitmaps +- improved handling of fragmented space using bitmaps - new mount option for debugging: fragment=data|metadata|all - updated balance filters: limit, stripes, usage - more bugfixes and cleanups @@ -1501,7 +1501,7 @@ Fixes: writes (now does: nologreplay) - default inline limit is now 2048 (instead of page size, usually 4096) -- /dev/btrfs-control now understands the GET_SUPPORTE_FEATURES ioctl +- /dev/btrfs-control now understands the GET_SUPPORTED_FEATURES ioctl - get rid of harmless message "''could not find root %llu''" - preparatory work for subpage-blocksize patchset - fix bug when using overlayfs @@ -1574,7 +1574,7 @@ Fixes: ^^^^^^^^^^^^^^^ - new tracepoints: file item -- fix qgoup accounting when inode_cache is in use +- fix qgroup accounting when inode_cache is in use - fix incorrect number report in stat::t_blocks under certain conditions - raid56 fixes: @@ -1877,7 +1877,7 @@ Fixes: ^^^^^^^^^^^^^^^ * ''fiemap'' exports information about shared extents -* bugfix and stability foucsed release +* bugfix and stability focused release 3.14 (Mar 2014) ^^^^^^^^^^^^^^^ diff --git a/Documentation/Send-receive.rst b/Documentation/Send-receive.rst index c0e8dee56a..d0a75e6a70 100644 --- a/Documentation/Send-receive.rst +++ b/Documentation/Send-receive.rst @@ -22,4 +22,4 @@ truncate), whole file operations (rename, delete). The stream can be sent over network, piped directly to the receive command or saved to a file. Each command in the stream is protected by a CRC32C checksum, with 0 as the initial value and no inversion. See :doc:`btrfs-send` and :doc:`btrfs-receive` for more, -for protocol description :doc:`dev-send-stream`. +for protocol description :doc:`dev/dev-send-stream`. diff --git a/Documentation/btrfs-filesystem.rst b/Documentation/btrfs-filesystem.rst index b036ae5eab..923032022f 100644 --- a/Documentation/btrfs-filesystem.rst +++ b/Documentation/btrfs-filesystem.rst @@ -140,7 +140,7 @@ defragment [options] | [|...] logic. Reasonable values are from tens to hundreds of megabytes. --step SIZE - Perform defragmention in the range in SIZE steps and flush (*-f*) after each one. + Perform defragmentation in the range in SIZE steps and flush (*-f*) after each one. The range is default (the whole file) or given by *-s* and *-l*, split into the steps or done in one go if the step is larger. Minimum range size is 256KiB. diff --git a/Documentation/btrfs-ioctl.rst b/Documentation/btrfs-ioctl.rst index 2a03ea3758..a3d1353e9d 100644 --- a/Documentation/btrfs-ioctl.rst +++ b/Documentation/btrfs-ioctl.rst @@ -550,7 +550,7 @@ BTRFS_IOC_ADD_DEV Add a given block device to the filesystem. Unlike the command :command:`btrfs device add` there's are no safety checks (like existence of another filesystem on the -device), device preparataion (like TRIM or zone reset), so use it with care. +device), device preparation (like TRIM or zone reset), so use it with care. This is a filesystem-exclusive operation and it will fail if there's another one already running, with one exception, when there's a paused balance. @@ -968,7 +968,7 @@ Use cases (:ref:`definition of constants`): * - ioctl args - :ref:`struct btrfs_ioctl_subvol_wait` * - args.subvolid - - Depending on the mode, the numeric id of subolume to wait for, or + - Depending on the mode, the numeric id of subvolume to wait for, or the one queried by *PEEK* modes * - args.mode - mode of operation described above diff --git a/Documentation/btrfs-man5.rst b/Documentation/btrfs-man5.rst index 56f9c939d6..fb85c93ff4 100644 --- a/Documentation/btrfs-man5.rst +++ b/Documentation/btrfs-man5.rst @@ -220,7 +220,7 @@ SYSFS INTERFACE .. include:: ch-sysfs.rst -.. _man-btrfs5-fileysstem-exclusive-operations: +.. _man-btrfs5-filesystem-exclusive-operations: FILESYSTEM EXCLUSIVE OPERATIONS ------------------------------- diff --git a/Documentation/btrfs-qgroup.rst b/Documentation/btrfs-qgroup.rst index b90cfbb540..0c5a009882 100644 --- a/Documentation/btrfs-qgroup.rst +++ b/Documentation/btrfs-qgroup.rst @@ -31,6 +31,12 @@ ownership. For example a fresh snapshot shares almost all the blocks with the original subvolume, new writes to either subvolume will raise towards the exclusive limit. +.. note:: + Qgroup limit only works when qgroup is in a consistent state. + If some workload marks qgroup inconsistent (like assigning a qgroup to another + qgroup), the limit will no longer work until the inconsistent flag is cleared + by :command:`btrfs quota rescan`. + The qgroup identifiers conform to *level/id* where level 0 is reserved to the qgroups associated with subvolumes. Such qgroups are created automatically. diff --git a/Documentation/ch-balance-filters.rst b/Documentation/ch-balance-filters.rst index 81a0ca8f03..46b8fb92cc 100644 --- a/Documentation/ch-balance-filters.rst +++ b/Documentation/ch-balance-filters.rst @@ -79,10 +79,26 @@ convert= Starting with kernel 4.5, the ``data`` chunks can be converted to/from the ``DUP`` profile on a single device. - .. note:: Starting with kernel 4.6, all profiles can be converted to/from ``DUP`` on multi-device filesystems. + .. warning:: + Bad or missing device are not detected immediately during + runtime and this depends on some later event like failed write + or failed transaction commit. If there's a known failing + device, or a device deleted by :file:`/sys/block//device/delete` interface, + the device will be still accessed and written to. + + In such case, one should not convert to a profile with lower + redundancy (e.g. from *RAID1* to *SINGLE*), + as attempts to create new chunks on the new devices will cause + various problems. + + The proper action is to use :command:`btrfs replace` or + :command:`btrfs device remove` to handle the failing/missing + device first. Then convert will work with all devices + correctly. + limit=, limit= Process only given number of chunks, after all filters are applied. This can be used to specifically target a chunk in connection with other filters (``drange``, diff --git a/Documentation/ch-sysfs.rst b/Documentation/ch-sysfs.rst index f58db400fc..1da273993f 100644 --- a/Documentation/ch-sysfs.rst +++ b/Documentation/ch-sysfs.rst @@ -5,26 +5,31 @@ The top level path is :file:`/sys/fs/btrfs/`, and the main directory layout is t ============================= =================================== ======== Relative Path Description Version ============================= =================================== ======== -features/ All supported features 3.14+ -/ Mounted fs UUID 3.14+ -/allocation/ Space allocation info 3.14+ -/features/ Features of the filesystem 3.14+ -/devices// Symlink to each block device sysfs 5.6+ -/devinfo// Btrfs specific info for each device 5.6+ -/qgroups/ Global qgroup info 5.9+ -/qgroups/_/ Info for each qgroup 5.9+ -/discard/ Discard stats and tunables 6.1+ +features/ All supported features 3.14 +/ Mounted fs UUID 3.14 +/allocation/ Space allocation info 3.14 +/bdi/ Backing device info (writeback) 5.9 +/devices// Symlink to each block device sysfs 5.6 +/devinfo// Btrfs specific info for each device 5.6 +/discard/ Discard stats and tunables 6.1 +/features/ Features of the filesystem 3.14 +/qgroups/ Global qgroup info 5.9 +/qgroups/_/ Info for each qgroup 5.9 ============================= =================================== ======== For :file:`/sys/fs/btrfs/features/` directory, each file means a supported feature -for the current kernel. +of the current kernel. Most files have value 0. Otherwise it depends on the file, +value *1* typically means the feature can be turned on a mounted filesystem. For :file:`/sys/fs/btrfs//features/` directory, each file means an enabled -feature for the mounted filesystem. +feature on the mounted filesystem. -The features shares the same name in section +The features share the same name in section :ref:`FILESYSTEM FEATURES`. +UUID +^^^^ + Files in :file:`/sys/fs/btrfs//` directory are: bg_reclaim_threshold @@ -72,7 +77,7 @@ exclusive_operation Shows the running exclusive operation. Check section - :ref:`FILESYSTEM EXCLUSIVE OPERATIONS` + :ref:`FILESYSTEM EXCLUSIVE OPERATIONS` for details. generation @@ -88,7 +93,7 @@ label metadata_uuid (RO, since: 5.0) - Shows the metadata uuid of the mounted filesystem. + Shows the metadata UUID of the mounted filesystem. Check `metadata_uuid` feature for more details. nodesize @@ -107,20 +112,24 @@ read_policy (RW, since: 5.11) Shows the current balance policy for reads. - Currently only "pid" (balance using pid value) is supported. + Currently only ``pid`` (balance using the process id (pid) value) is + supported. More balancing policies are available in experimental + build, namely round-robin. sectorsize (RO, since: 3.14) Shows the sectorsize of the mounted filesystem. - temp_fsid (RO, since 6.7) Indicate that this filesystem got assigned a temporary FSID at mount time, making possible to mount devices with the same FSID. +UUID/allocations +^^^^^^^^^^^^^^^^ + Files and directories in :file:`/sys/fs/btrfs//allocations` directory are: global_rsv_reserved @@ -136,8 +145,10 @@ global_rsv_size `data/`, `metadata/` and `system/` directories (RO, since: 5.14) - Space info accounting for the 3 chunk types. - Mostly for debug purposes. + Space info accounting for the 3 block group types. + +UUID/allocations/{data,metadata,system} +""""""""""""""""""""""""""""""""""""""" Files in :file:`/sys/fs/btrfs//allocations/{data,metadata,system}` directory are: @@ -152,13 +163,20 @@ bytes_* (RO) Values of the corresponding data structures for the given block group - type and profile. + type and profile that are used internally and may change rapidly depending + on the load. + + Complete list: bytes_may_use, bytes_pinned, bytes_readonly, + bytes_reserved, bytes_used, bytes_zone_unusable chunk_size (RW, since: 6.0) - Shows the chunk size. Can be changed for data and metadata. - Cannot be set for zoned devices. + Shows the chunk size. Can be changed for data and metadata (independently) + and cannot be set for system block group type. + Cannot be set for zoned devices as it depends on the fixed device zone size. + Upper bound is 10% of the filesystem size, the value must be multiple of 256MiB + and greater than 0. size_classes (RO, since: 6.3) @@ -173,6 +191,27 @@ size_classes medium 282 large 93 +UUID/bdi +^^^^^^^^ + +Symlink to the sysfs directory of the backing device info (BDI), which is +related to writeback process and infrastructure. + +UUID/devices +^^^^^^^^^^^^ + +Files in :file:`/sys/fs/btrfs//devices` directory are symlinks named +after device nodes (e.g. sda, dm-0) and pointing to their sysfs directory. + +UUID/devinfo +^^^^^^^^^^^^ + +The directory contains subdirectories named after device ids (numeric values). Each +subdirectory has information about the device of the given *devid*. + +UUID/devinfo/DEVID +"""""""""""""""""" + Files in :file:`/sys/fs/btrfs//devinfo/` directory are: error_stats: @@ -192,44 +231,47 @@ fsid: (RO, since: 5.17) Shows the fsid which the device belongs to. - It can be different than the `` if it's a seed device. + It can be different than the ``UUID`` if it's a seed device. in_fs_metadata (RO, since: 5.6) Shows whether we have found the device. - Should always be 1, as if this turns to 0, the `` directory + Should always be 1, as if this turns to 0, the :file:`DEVID` directory would get removed automatically. missing (RO, since: 5.6) - Shows whether the device is missing. + Shows whether the device is considered missing by the kernel module. replace_target (RO, since: 5.6) Shows whether the device is the replace target. - If no dev-replace is running, this value should be 0. + If no device replace is running, this value is 0. scrub_speed_max (RW, since: 5.14) Shows the scrub speed limit for this device. The unit is Bytes/s. - 0 means no limit. The value can be set but is not persisent. + 0 means no limit. The value can be set but is not persistent. writeable (RO, since: 5.6) Show if the device is writeable. +UUID/qgroups +^^^^^^^^^^^^ + Files in :file:`/sys/fs/btrfs//qgroups/` directory are: enabled (RO, since: 6.1) Shows if qgroup is enabled. - Also, if qgroup is disabled, the `qgroups` directory would + Also, if qgroup is disabled, the :file:`qgroups` directory will be removed automatically. inconsistent @@ -249,12 +291,17 @@ drop_subtree_threshold we will not trigger qgroup account at all, but mark qgroup inconsistent to avoid the huge workload. - Default value is 8, where no subtree drop can trigger qgroup. + Default value is 3, which means that trees of low height will be accounted + properly as this is sufficiently fast. The value was 8 until 6.13 where + no subtree drop can trigger qgroup rescan making it less useful. Lower value can reduce qgroup workload, at the cost of extra qgroup rescan to re-calculate the numbers. -Files in :file:`/sys/fs/btrfs//_/` directory are: +UUID/qgroups/LEVEL_ID +""""""""""""""""""""" + +Files in each :file:`/sys/fs/btrfs//qgroups/_/` directory are: exclusive (RO, since: 5.9) @@ -297,6 +344,9 @@ rsv_meta_prealloc Shows the reserved bytes for preallocated metadata. +UUID/discard +^^^^^^^^^^^^ + Files in :file:`/sys/fs/btrfs//discard/` directory are: discardable_bytes diff --git a/Documentation/dev/CmdLineConventions.rst b/Documentation/dev/CmdLineConventions.rst index 72ac875ca7..1c3034b204 100644 --- a/Documentation/dev/CmdLineConventions.rst +++ b/Documentation/dev/CmdLineConventions.rst @@ -58,7 +58,7 @@ or change descriptions. * brief but descriptive * long and descriptive can be used in justified cases (e.g. conversion options - in :doc:`btrfstune` because of the single command without subcommands) + in :doc:`../btrfstune` because of the single command without subcommands) Option parameters ^^^^^^^^^^^^^^^^^ diff --git a/Documentation/dev/Developer-s-FAQ.rst b/Documentation/dev/Developer-s-FAQ.rst index 3b33207d10..f258a6cca8 100644 --- a/Documentation/dev/Developer-s-FAQ.rst +++ b/Documentation/dev/Developer-s-FAQ.rst @@ -64,7 +64,7 @@ candidate for a CC: of the bugreport. **Do**: talk to the maintainer if he forgot to add this tag to the final patch. Reviews do take time and the patches land in various branches early after -they're sent to the mailingslist for testing, but the reviews are always +they're sent to the mailinglist for testing, but the reviews are always welcome. **Do**: collect the Reviewed-by tags for patches that get resent unchanged e.g. @@ -236,7 +236,7 @@ Good practices, contribution hints - be patient if your patch is not accepted immediately, try to send a gentle ping if there's a significant time without any action - if you want to start contributing but are not sure about how to do that, - lurk in the mailingist or on the IRC channel + lurk in the mailinglist or on the IRC channel - every patch should implement one thing -- this is vaguely defined, you may receive comments about patch splitting or merging with other - every patch must be compilable when applied, possibly with all related @@ -451,7 +451,7 @@ copyright holders of changes in a given file. The code is usually heavily changed over time in smaller portions, slowly morphing into something that does not resemble the original code anymore though it shares a lot of the core ideas and implemented logic. A copyright notice by a company that does not exist -anymore from 10 years ago is a clear example of uselesness for the developers. +anymore from 10 years ago is a clear example of uselessness for the developers. When code is moved verbatim from a file to another file, in the new file it appears to be contributed by a single author while it is in most cases code diff --git a/Documentation/dev/Development-notes.rst b/Documentation/dev/Development-notes.rst index 76c492caeb..65b7af4e47 100644 --- a/Documentation/dev/Development-notes.rst +++ b/Documentation/dev/Development-notes.rst @@ -120,7 +120,7 @@ There are several ways how to react to the unexpected conditions: This would cause the filesystem to be flipped read-only to prevent further corruption. - Additionally call trace would be dumpped for the first btrfs_abort_transaction() + Additionally call trace would be dumped for the first btrfs_abort_transaction() call site. - ASSERT() diff --git a/Documentation/dev/dev-btrees.rst b/Documentation/dev/dev-btrees.rst index 1a710f8815..83e090fc1c 100644 --- a/Documentation/dev/dev-btrees.rst +++ b/Documentation/dev/dev-btrees.rst @@ -114,7 +114,7 @@ Log Root Tree UUID Tree --------- -The tree storesc correspondence between UUIDs and subvolumes. Used for +The tree stores correspondence between UUIDs and subvolumes. Used for quick lookup during send. diff --git a/Documentation/dev/dev-json.rst b/Documentation/dev/dev-json.rst index 09d8533139..4983810585 100644 --- a/Documentation/dev/dev-json.rst +++ b/Documentation/dev/dev-json.rst @@ -3,12 +3,17 @@ JSON output Supported types: +* any: a valid *printf* format, parameters to *fmt_print()* must match the number (and are not validated) * number: ``%llu`` * string: ``%s`` -* string: ``str`` (escaped special characters) -* bool: ``bool`` (unquoted native json value) -* qgroupid: ``qgroupid`` (split to 48/16 for level/subvolid) -* size: ``size`` +* string: ``str`` - backslash escaped special characters (0x08, 0x08, 0x0a, 0x0c, 0x0d, 0x0b), + the rest of range from *0x00* to *0x1f* as *\\uXXXX* encoding +* bool: ``bool`` - unquoted native json values *true* or *false* +* qgroupid: ``qgroupid`` - split to 48/16 for level/subvolid +* size: ``size`` - size with SI/IEC size suffix +* size: ``size-or-none`` - same as *size* but for 0 it's *none* +* UUID: ``uuid`` - if all zeros then *null* (native json), or properly formatted UUID string +* date + time: ``date-time`` - timestamp formatted as *YYYY-MM-DD HH:MM:SS TIMEZONE* Commands that support json output --------------------------------- @@ -23,12 +28,18 @@ Commands that support json output Recommendations --------------- +Keys and formatting are defined as an array of *struct rowspec*. + * key names - * should be unified if they mean the same thing + * should be unified with the printed value if they mean the same thing * not abbreviated (e.g. *generation* instead of *gen*) * referring to existing and well known names (qgroupid, devid, ...) + * using ``-`` as word separator, or ``_`` if it's better to keep the same name of the value * values * numbers without suffix or other transformation, i.e. no *KiB* + * formatted by the types if possible + * any *printf* format is possible but should be avoided or a new type should + be defined * printing more data about an item is better than printing less, assuming the filtering is done on the user side * structure of json output may not reflect the way it's printed in plain text, @@ -36,3 +47,5 @@ Recommendations * if plain and json output roughly follow the same style, e.g. line oriented that is easy to transform to a map, then both outputs should use the same rowspec +* one value can be printed by multiple rowspecs that may do different + formatting depending on the context diff --git a/Documentation/mkfs.btrfs.rst b/Documentation/mkfs.btrfs.rst index f58c2fe62a..9d8cfe955e 100644 --- a/Documentation/mkfs.btrfs.rst +++ b/Documentation/mkfs.btrfs.rst @@ -150,6 +150,9 @@ OPTIONS Populate the toplevel subvolume with files from *rootdir*. This does not require root permissions to write the new files or to mount the filesystem. + Directories can be created as subvolumes, see also option *--subvol*. + Hardlinks are detected and created in the filesystem image. + .. note:: This option may enlarge the image or file to ensure it's big enough to contain the files from *rootdir*. Since version 4.14.1 the filesystem size is @@ -160,6 +163,10 @@ OPTIONS *no* (the default), *zstd*, *lzo* or *zlib*. The optional value *level* is a compression level, 1..15 for *zstd*, 1..9 for *zlib*. + It is recommended to use the highest level to achieve maximum space savings. + Compression at mkfs time is not as constrained as in kernel where it's + desirable to use the less CPU load. Otherwise the default level is 3. + As with the kernel, :command:`mkfs.btrfs` won't write compressed extents when they would be larger than the uncompressed versions, and will set file attribute *NOCOMPRESS* if its beginning is found to be incompressible. @@ -173,7 +180,7 @@ OPTIONS directory. The option *--rootdir* must also be specified, and *subdir* must be an existing subdirectory within it. This option can be specified multiple times. - *type* is an optional additional modifier. Valid choices are: + The *type* is an optional additional modifier. Valid choices are: * *default*: create as default subvolume * *ro*: create as read-only subvolume @@ -183,14 +190,16 @@ OPTIONS Only one of *default* and *default-ro* may be specified. If you wish to create a subvolume with a name containing a colon and you don't - want this to be parsed as containing a modifier, you can prefix the path with `./`: + want this to be parsed as containing a modifier, you can prefix the path with :file:`./`: .. code-block:: bash $ mkfs.btrfs --rootdir dir --subvol ./ro:subdir /dev/loop0 - If there are hard links inside *rootdir* and *subdir* will split the - subvolumes, like the following case:: + If there are hardlinks inside *rootdir* and *subdir* will split the + subvolumes, like the following case: + + .. code-block:: none rootdir/ |- hardlink1 @@ -198,9 +207,9 @@ OPTIONS |- subdir/ <- will be a subvolume |- hardlink3 - In that case we cannot create `hardlink3` as hardlinks of - `hardlink1` and `hardlink2` because hardlink3 will be inside a new - subvolume. + In that case we cannot create :file:`hardlink3` as hardlinks of + :file:`hardlink1` and :file:`hardlink2` because :file:`hardlink3` will + be inside a new subvolume. --shrink Shrink the filesystem to its minimal size, only works with *--rootdir* option. @@ -605,7 +614,7 @@ hopefully not destroying both copies of particular data in case of DUP. The wear levelling techniques can also lead to reduced redundancy, even if the device does not do any deduplication. The controllers may put data written in -a short timespan into the same physical storage unit (cell, block etc). In case +a short time span into the same physical storage unit (cell, block etc). In case this unit dies, both copies are lost. BTRFS does not add any artificial delay between metadata writes. diff --git a/README.md b/README.md index 055c11e888..e93d29c6b8 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Btrfs-progs [![codecov](https://codecov.io/gh/kdave/btrfs-progs/branch/coverage-test/graph/badge.svg?token=fhLI8V9s0k)](https://codecov.io/gh/kdave/btrfs-progs) [![static](https://github.com/kdave/btrfs-progs/actions/workflows/artifacts-static-build.yml/badge.svg)](https://github.com/kdave/btrfs-progs/actions/workflows/artifacts-static-build.yml) [![release](https://github.com/kdave/btrfs-progs/actions/workflows/ci-build-test.yml/badge.svg)](https://github.com/kdave/btrfs-progs/actions/workflows/ci-build-test.yml) +[![codeql](https://github.com/kdave/btrfs-progs/actions/workflows/codeql.yml/badge.svg)](https://github.com/kdave/btrfs-progs/actions/workflows/codeql.yml) Userspace utilities to manage btrfs filesystems. License: GPLv2. diff --git a/ci/images/ci-centos-7-x86_64/Dockerfile b/ci/images/ci-centos-7-x86_64/Dockerfile index 6b49055ddb..c4e3bcd5ef 100644 --- a/ci/images/ci-centos-7-x86_64/Dockerfile +++ b/ci/images/ci-centos-7-x86_64/Dockerfile @@ -29,7 +29,7 @@ COPY ./test-build . COPY ./run-tests . COPY ./devel.tar.gz . -CMD ./test-build devel --disable-documentation --disable-libudev +CMD ["./test-build", "devel", "--disable-documentation", "--disable-libudev"] # Continue with: # cd /tmp diff --git a/ci/images/ci-centos-8-x86_64/Dockerfile b/ci/images/ci-centos-8-x86_64/Dockerfile index b10f51126e..d36ad510fe 100644 --- a/ci/images/ci-centos-8-x86_64/Dockerfile +++ b/ci/images/ci-centos-8-x86_64/Dockerfile @@ -32,7 +32,7 @@ COPY ./test-build . COPY ./run-tests . COPY ./devel.tar.gz . -CMD ./test-build devel --disable-documentation --disable-libudev --disable-zoned +CMD ["./test-build", "devel", "--disable-documentation", "--disable-libudev", "--disable-zoned"] # Continue with: # cd /tmp diff --git a/ci/images/ci-musl-i386/Dockerfile b/ci/images/ci-musl-i386/Dockerfile index 912d2beb6c..f6248d4f62 100644 --- a/ci/images/ci-musl-i386/Dockerfile +++ b/ci/images/ci-musl-i386/Dockerfile @@ -16,4 +16,4 @@ RUN apk add wget COPY ./test-build . COPY ./devel.tar.gz . -CMD ./test-build devel --disable-documentation --disable-backtrace --disable-libudev +CMD ["./test-build", "devel", "--disable-documentation", "--disable-backtrace", "--disable-libudev"] diff --git a/ci/images/ci-musl-x86_64/Dockerfile b/ci/images/ci-musl-x86_64/Dockerfile index 912d2beb6c..f6248d4f62 100644 --- a/ci/images/ci-musl-x86_64/Dockerfile +++ b/ci/images/ci-musl-x86_64/Dockerfile @@ -16,4 +16,4 @@ RUN apk add wget COPY ./test-build . COPY ./devel.tar.gz . -CMD ./test-build devel --disable-documentation --disable-backtrace --disable-libudev +CMD ["./test-build", "devel", "--disable-documentation", "--disable-backtrace", "--disable-libudev"] diff --git a/ci/images/ci-openSUSE-Leap-15.3-x86_64/Dockerfile b/ci/images/ci-openSUSE-Leap-15.3-x86_64/Dockerfile index 5fecce0fbb..49d00d5d2c 100644 --- a/ci/images/ci-openSUSE-Leap-15.3-x86_64/Dockerfile +++ b/ci/images/ci-openSUSE-Leap-15.3-x86_64/Dockerfile @@ -26,7 +26,7 @@ COPY ./run-tests . COPY ./devel.tar.gz . # The blkzoned.h exists but blk_zone.capacity is missing, disable zoned mode explicitly -CMD ./test-build devel --disable-documentation --disable-zoned +CMD ["./test-build", "devel", "--disable-documentation", "--disable-zoned"] # Continue with: # cd /tmp diff --git a/ci/images/ci-openSUSE-Leap-15.6-x86_64/Dockerfile b/ci/images/ci-openSUSE-Leap-15.6-x86_64/Dockerfile index df8af2466d..2a4fedecf3 100644 --- a/ci/images/ci-openSUSE-Leap-15.6-x86_64/Dockerfile +++ b/ci/images/ci-openSUSE-Leap-15.6-x86_64/Dockerfile @@ -26,7 +26,7 @@ COPY ./run-tests . COPY ./devel.tar.gz . # The blkzoned.h exists but blk_zone.capacity is missing, disable zoned mode explicitly -CMD ./test-build devel --disable-documentation --disable-zoned +CMD ["./test-build", "devel", "--disable-documentation", "--disable-zoned"] # Continue with: # cd /tmp diff --git a/ci/images/ci-openSUSE-tumbleweed-x86_64/Dockerfile b/ci/images/ci-openSUSE-tumbleweed-x86_64/Dockerfile index 554745de65..d6ecde420f 100644 --- a/ci/images/ci-openSUSE-tumbleweed-x86_64/Dockerfile +++ b/ci/images/ci-openSUSE-tumbleweed-x86_64/Dockerfile @@ -33,7 +33,7 @@ COPY ./test-build . COPY ./run-tests . COPY ./devel.tar.gz . -CMD ./test-build devel --disable-documentation +CMD ["./test-build", "devel", "--disable-documentation"] # Continue with: # cd /tmp diff --git a/ci/images/ci-rockylinux-9-x86_64/Dockerfile b/ci/images/ci-rockylinux-9-x86_64/Dockerfile index 7c1651caf4..3328dfdbb6 100644 --- a/ci/images/ci-rockylinux-9-x86_64/Dockerfile +++ b/ci/images/ci-rockylinux-9-x86_64/Dockerfile @@ -29,7 +29,7 @@ COPY ./test-build . COPY ./run-tests . COPY ./devel.tar.gz . -CMD ./test-build devel --disable-documentation +CMD ["./test-build", "devel", "--disable-documentation"] # Continue with: # cd /tmp diff --git a/cmds/qgroup.c b/cmds/qgroup.c index a612130c94..45439953be 100644 --- a/cmds/qgroup.c +++ b/cmds/qgroup.c @@ -36,6 +36,7 @@ #include "kernel-shared/uapi/btrfs.h" #include "kernel-shared/ctree.h" #include "common/open-utils.h" +#include "common/sysfs-utils.h" #include "common/utils.h" #include "common/help.h" #include "common/units.h" @@ -1610,6 +1611,33 @@ static void print_all_qgroups_json(struct qgroup_lookup *qgroup_lookup) fmt_end(&fctx); } +/* + * Tree search based inconsistent flag is only updated at transaction commit + * time. + * Thus even if the qgroup_status flag shows consistent, the qgroup may already + * be in an inconsistent status. + */ +static void check_qgroup_sysfs_inconsistent(int fd, + const struct qgroup_lookup *qgroup_lookup) +{ + u64 value; + int sysfs_fd; + int ret; + + if (qgroup_lookup->flags & BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT) + return; + sysfs_fd = sysfs_open_fsid_file(fd, "qgroups/inconsistent"); + if (fd < 0) + return; + ret = sysfs_read_fsid_file_u64(fd, "qgroups/inconsistent", &value); + if (ret < 0) + goto out; + if (value) + warning("qgroup data inconsistent, rescan recommended"); +out: + close(sysfs_fd); +} + static int show_qgroups(int fd, struct btrfs_qgroup_filter_set *filter_set, struct btrfs_qgroup_comparer_set *comp_set) @@ -1622,6 +1650,8 @@ static int show_qgroups(int fd, ret = qgroups_search_all(fd, &qgroup_lookup); if (ret) return ret; + + check_qgroup_sysfs_inconsistent(fd, &qgroup_lookup); __filter_and_sort_qgroups(&qgroup_lookup, &sort_tree, filter_set, comp_set); if (bconf.output_format == CMD_FORMAT_JSON) diff --git a/cmds/scrub.c b/cmds/scrub.c index 23af162809..b1d2f731b1 100644 --- a/cmds/scrub.c +++ b/cmds/scrub.c @@ -204,9 +204,10 @@ static void print_scrub_summary(struct btrfs_scrub_progress *p, struct scrub_sta pr_verbose(LOG_DEFAULT, "Total to scrub: %s\n", pretty_size_mode(bytes_total, unit_mode)); } + /* * Rate and size units are disproportionate so they are affected only - * by --raw, otherwise it's human readable + * by --raw, otherwise it's human readable (respecting the SI or IEC mode). */ if (unit_mode == UNITS_RAW) { pr_verbose(LOG_DEFAULT, "Rate: %s/s", @@ -218,11 +219,16 @@ static void print_scrub_summary(struct btrfs_scrub_progress *p, struct scrub_sta pr_verbose(LOG_DEFAULT, " (some device limits set)"); pr_verbose(LOG_DEFAULT, "\n"); } else { + unsigned int mode = UNITS_HUMAN_DECIMAL; + + if (unit_mode & UNITS_BINARY) + mode = UNITS_HUMAN_BINARY; + pr_verbose(LOG_DEFAULT, "Rate: %s/s", - pretty_size_mode(bytes_per_sec, unit_mode)); + pretty_size_mode(bytes_per_sec, mode)); if (limit > 1) pr_verbose(LOG_DEFAULT, " (limit %s/s)", - pretty_size_mode(limit, unit_mode)); + pretty_size_mode(limit, mode)); else if (limit == 1) pr_verbose(LOG_DEFAULT, " (some device limits set)"); pr_verbose(LOG_DEFAULT, "\n"); diff --git a/common/units.h b/common/units.h index d93977ff2a..4a638e7f06 100644 --- a/common/units.h +++ b/common/units.h @@ -31,9 +31,9 @@ #define UNITS_TBYTES (5) #define UNITS_RAW (1U << UNITS_MODE_SHIFT) #define UNITS_BINARY (2U << UNITS_MODE_SHIFT) -#define UNITS_DECIMAL (3U << UNITS_MODE_SHIFT) +#define UNITS_DECIMAL (4U << UNITS_MODE_SHIFT) /* Interpret the u64 value as s64 */ -#define UNITS_NEGATIVE (4U << UNITS_MODE_SHIFT) +#define UNITS_NEGATIVE (8U << UNITS_MODE_SHIFT) #define UNITS_MODE_MASK ((1U << UNITS_MODE_SHIFT) - 1) #define UNITS_MODE_SHIFT (8) #define UNITS_HUMAN_BINARY (UNITS_BINARY) diff --git a/kernel-shared/ctree.h b/kernel-shared/ctree.h index 6ba982ef5f..8c923be967 100644 --- a/kernel-shared/ctree.h +++ b/kernel-shared/ctree.h @@ -831,6 +831,16 @@ static inline u64 btrfs_dev_stats_value(const struct extent_buffer *eb, return val; } +static inline void btrfs_set_dev_stats_value(struct extent_buffer *eb, + struct btrfs_dev_stats_item *ptr, + int index, u64 val) +{ + write_extent_buffer(eb, &val, + offsetof(struct btrfs_dev_stats_item, values) + + ((unsigned long)ptr) + (index * sizeof(u64)), + sizeof(val)); +} + /* struct btrfs_ioctl_search_header */ static inline u64 btrfs_search_header_transid(struct btrfs_ioctl_search_header *sh) { diff --git a/kernel-shared/volumes.c b/kernel-shared/volumes.c index b21231efe8..a0842d58d2 100644 --- a/kernel-shared/volumes.c +++ b/kernel-shared/volumes.c @@ -1135,6 +1135,7 @@ int btrfs_add_device(struct btrfs_trans_handle *trans, int ret; struct btrfs_path *path; struct btrfs_dev_item *dev_item; + struct btrfs_dev_stats_item *dev_stats; struct extent_buffer *leaf; struct btrfs_key key; struct btrfs_root *root = fs_info->chunk_root; @@ -1149,6 +1150,7 @@ int btrfs_add_device(struct btrfs_trans_handle *trans, if (ret) goto out; + /* Add DEV_ITEM. */ key.objectid = BTRFS_DEV_ITEMS_OBJECTID; key.type = BTRFS_DEV_ITEM_KEY; key.offset = free_devid; @@ -1182,6 +1184,27 @@ int btrfs_add_device(struct btrfs_trans_handle *trans, BTRFS_UUID_SIZE); btrfs_mark_buffer_dirty(leaf); fs_info->fs_devices->total_rw_bytes += device->total_bytes; + + btrfs_release_path(path); + + /* Add DEV STATS item. */ + key.objectid = BTRFS_DEV_STATS_OBJECTID; + key.type = BTRFS_PERSISTENT_ITEM_KEY; + key.offset = free_devid; + + ret = btrfs_insert_empty_item(trans, fs_info->dev_root, path, &key, + sizeof(*dev_stats)); + if (ret) + goto out; + + leaf = path->nodes[0]; + dev_stats = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_stats_item); + + for (int i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) + btrfs_set_dev_stats_value(leaf, dev_stats, i, 0); + + btrfs_mark_buffer_dirty(leaf); + ret = 0; out: diff --git a/mkfs/common.c b/mkfs/common.c index f6501a1306..bb5a2ad46f 100644 --- a/mkfs/common.c +++ b/mkfs/common.c @@ -127,11 +127,13 @@ static int btrfs_create_tree_root(int fd, struct btrfs_mkfs_config *cfg, memcpy(root_item.uuid, uuid, BTRFS_UUID_SIZE); btrfs_set_stack_timespec_sec(&root_item.otime, now); btrfs_set_stack_timespec_sec(&root_item.ctime, now); + btrfs_set_stack_inode_flags(inode_item, BTRFS_INODE_ROOT_ITEM_INIT); } else { memset(uuid, 0, BTRFS_UUID_SIZE); memcpy(root_item.uuid, uuid, BTRFS_UUID_SIZE); btrfs_set_stack_timespec_sec(&root_item.otime, 0); btrfs_set_stack_timespec_sec(&root_item.ctime, 0); + btrfs_set_stack_inode_flags(inode_item, 0); } write_extent_buffer(buf, &root_item, btrfs_item_ptr_offset(buf, nritems), @@ -367,6 +369,7 @@ int make_btrfs(int fd, struct btrfs_mkfs_config *cfg) struct btrfs_chunk *chunk; struct btrfs_dev_item *dev_item; struct btrfs_dev_extent *dev_extent; + struct btrfs_dev_stats_item *dev_stats; enum btrfs_mkfs_block blocks[MKFS_BLOCK_COUNT]; u8 chunk_tree_uuid[BTRFS_UUID_SIZE]; u8 *ptr; @@ -671,8 +674,25 @@ int make_btrfs(int fd, struct btrfs_mkfs_config *cfg) memset(buf->data + sizeof(struct btrfs_header), 0, cfg->nodesize - sizeof(struct btrfs_header)); nritems = 0; - itemoff = cfg->leaf_data_size - sizeof(struct btrfs_dev_extent); + itemoff = cfg->leaf_data_size; + + /* Add a DEV_STATS item for device 1. */ + itemoff -= sizeof(struct btrfs_dev_stats_item); + btrfs_set_disk_key_objectid(&disk_key, BTRFS_DEV_STATS_OBJECTID); + btrfs_set_disk_key_type(&disk_key, BTRFS_PERSISTENT_ITEM_KEY); + btrfs_set_disk_key_offset(&disk_key, 1); + btrfs_set_item_key(buf, &disk_key, nritems); + btrfs_set_item_offset(buf, nritems, itemoff); + btrfs_set_item_size(buf, nritems, sizeof(struct btrfs_dev_stats_item)); + dev_stats = btrfs_item_ptr(buf, nritems, struct btrfs_dev_stats_item); + + for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) + btrfs_set_dev_stats_value(buf, dev_stats, i, 0); + + nritems++; + /* Add the DEV_EXTENT item for the system chunk. */ + itemoff -= sizeof(struct btrfs_dev_extent); btrfs_set_disk_key_objectid(&disk_key, 1); btrfs_set_disk_key_offset(&disk_key, system_group_offset); btrfs_set_disk_key_type(&disk_key, BTRFS_DEV_EXTENT_KEY); diff --git a/mkfs/main.c b/mkfs/main.c index 4191d04d75..92441710fd 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -902,7 +902,8 @@ static int setup_quota_root(struct btrfs_fs_info *fs_info) } ret = btrfs_create_root(trans, fs_info, BTRFS_QUOTA_TREE_OBJECTID); if (ret < 0) { - error("failed to create quota root: %d (%m)", ret); + errno = -ret; + error("failed to create quota root: %m"); goto fail; } quota_root = fs_info->quota_root; @@ -914,7 +915,8 @@ static int setup_quota_root(struct btrfs_fs_info *fs_info) ret = btrfs_insert_empty_item(trans, quota_root, &path, &key, sizeof(*qsi)); if (ret < 0) { - error("failed to insert qgroup status item: %d (%m)", ret); + errno = -ret; + error("failed to insert qgroup status item: %m"); goto fail; } @@ -938,7 +940,8 @@ static int setup_quota_root(struct btrfs_fs_info *fs_info) /* Currently mkfs will only create one subvolume */ ret = insert_qgroup_items(trans, fs_info, BTRFS_FS_TREE_OBJECTID); if (ret < 0) { - error("failed to insert qgroup items: %d (%m)", ret); + errno = -ret; + error("failed to insert qgroup items: %m"); goto fail; } @@ -953,7 +956,8 @@ static int setup_quota_root(struct btrfs_fs_info *fs_info) if (simple) { ret = touch_root_subvol(fs_info); if (ret) { - error("failed to touch root dir for simple quota accounting %d (%m)", ret); + errno = -ret; + error("failed to touch root dir for simple quota accounting: %m"); goto fail; } } @@ -964,12 +968,15 @@ static int setup_quota_root(struct btrfs_fs_info *fs_info) */ ret = qgroup_verify_all(fs_info); if (ret < 0) { - error("qgroup rescan failed: %d (%m)", ret); + errno = -ret; + error("qgroup rescan failed: %m"); return ret; } ret = repair_qgroups(fs_info, &qgroup_repaired, true); - if (ret < 0) - error("failed to fill qgroup info: %d (%m)", ret); + if (ret < 0) { + errno = -ret; + error("failed to fill qgroup info: %m"); + } return ret; fail: btrfs_abort_transaction(trans, ret); @@ -1142,7 +1149,7 @@ static int parse_subvolume(const char *path, struct list_head *subvols, if (subvol->is_default) { if (*has_default_subvol) { - error("default subvol can only be specified once"); + error("default subvolume can only be specified once"); free(subvol); return 1; } @@ -1453,25 +1460,25 @@ int BOX_MAIN(mkfs)(int argc, char **argv) struct rootdir_subvol *rds2; if (path_cat_out(path, source_dir, rds->dir)) { - error("path invalid"); + error("path invalid: %s", path); ret = 1; goto error; } if (!realpath(path, rds->full_path)) { - error("could not get canonical path to %s", rds->dir); + error("could not get canonical path: %s", rds->dir); ret = 1; goto error; } if (!path_exists(rds->full_path)) { - error("subvolume %s does not exist", rds->dir); + error("subvolume path does not exist: %s", rds->dir); ret = 1; goto error; } if (!path_is_dir(rds->full_path)) { - error("subvolume %s is not a directory", rds->dir); + error("subvolume is not a directory: %s", rds->dir); ret = 1; goto error; } @@ -1486,7 +1493,7 @@ int BOX_MAIN(mkfs)(int argc, char **argv) rds2 != rds; rds2 = list_next_entry(rds2, list)) { if (strcmp(rds2->full_path, rds->full_path) == 0) { - error("subvolume %s specified more than once", rds->dir); + error("subvolume specified more than once: %s", rds->dir); ret = 1; goto error; } @@ -1658,6 +1665,10 @@ int BOX_MAIN(mkfs)(int argc, char **argv) goto error; } + /* This is also fixed in kernel, but the flag has no real meaning anymore. */ + if (nodesize > sysconf(_SC_PAGE_SIZE)) + features.incompat_flags |= BTRFS_FEATURE_INCOMPAT_BIG_METADATA; + if (sectorsize < sizeof(struct btrfs_super_block)) { error("sectorsize smaller than superblock: %u < %zu", sectorsize, sizeof(struct btrfs_super_block)); @@ -1897,14 +1908,16 @@ int BOX_MAIN(mkfs)(int argc, char **argv) ret = create_metadata_block_groups(root, mixed, &allocation); if (ret) { - error("failed to create default block groups: %d", ret); + errno = -ret; + error("failed to create default block groups: %m"); goto error; } if (features.incompat_flags & BTRFS_FEATURE_INCOMPAT_RAID_STRIPE_TREE) { ret = setup_raid_stripe_tree_root(fs_info); if (ret < 0) { - error("failed to initialize raid-stripe-tree: %d (%m)", ret); + errno = -ret; + error("failed to initialize raid-stripe-tree: %m"); goto out; } } @@ -1919,21 +1932,24 @@ int BOX_MAIN(mkfs)(int argc, char **argv) ret = create_data_block_groups(trans, root, mixed, &allocation); if (ret) { - error("failed to create default data block groups: %d", ret); + errno = -ret; + error("failed to create default data block groups: %m"); goto error; } if (features.incompat_flags & BTRFS_FEATURE_INCOMPAT_EXTENT_TREE_V2) { ret = create_global_roots(trans, nr_global_roots); if (ret) { - error("failed to create global roots: %d", ret); + errno = -ret; + error("failed to create global roots: %m"); goto error; } } ret = make_root_dir(trans, root); if (ret) { - error("failed to setup the root directory: %d", ret); + errno = -ret; + error("failed to setup the root directory: %m"); goto error; } @@ -1976,8 +1992,9 @@ int BOX_MAIN(mkfs)(int argc, char **argv) prepare_ctx[i].file, dev_byte_count, sectorsize, sectorsize, sectorsize); if (ret) { - error("unable to add %s to filesystem: %d", - prepare_ctx[i].file, ret); + errno = -ret; + error("unable to add %s to filesystem: %m", + prepare_ctx[i].file); goto error; } if (bconf.verbose >= LOG_INFO) { @@ -1996,7 +2013,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv) ret = create_raid_groups(trans, root, data_profile, metadata_profile, mixed, &allocation); if (ret) { - error("unable to create raid groups: %d", ret); + errno = -ret; + error("unable to create raid groups: %m"); goto out; } @@ -2031,7 +2049,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv) ret = btrfs_make_subvolume(trans, BTRFS_DATA_RELOC_TREE_OBJECTID, false); if (ret) { - error("unable to create data reloc tree: %d", ret); + errno = -ret; + error("unable to create data reloc tree: %m"); goto out; } @@ -2061,11 +2080,21 @@ int BOX_MAIN(mkfs)(int argc, char **argv) pretty_size_mode(compression_level, UNITS_RAW) : ""); + /* Print subvolumes now as btrfs_mkfs_fill_dir() deletes the list. */ + list_for_each_entry(rds, &subvols, list) { + pr_verbose(LOG_DEFAULT, " Subvolume (%s%s): %s%s\n", + rds->is_default ? "d" : "", + rds->readonly ? "ro" : "rw", + rds->is_default ? "" : " ", + rds->dir); + } + ret = btrfs_mkfs_fill_dir(trans, source_dir, root, &subvols, compression, compression_level); if (ret) { - error("error while filling filesystem: %d", ret); + errno = -ret; + error("error while filling filesystem: %m"); btrfs_abort_transaction(trans, ret); goto out; } @@ -2077,18 +2106,13 @@ int BOX_MAIN(mkfs)(int argc, char **argv) goto out; } - list_for_each_entry(rds, &subvols, list) { - pr_verbose(LOG_DEFAULT, " Subvolume: %s\n", - rds->full_path); - } - if (shrink_rootdir) { pr_verbose(LOG_DEFAULT, " Shrink: yes\n"); ret = btrfs_mkfs_shrink_fs(fs_info, &shrink_size, shrink_rootdir); if (ret < 0) { - error("error while shrinking filesystem: %d", - ret); + errno = -ret; + error("error while shrinking filesystem: %m"); goto out; } } else { @@ -2103,7 +2127,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv) ret = cleanup_temp_chunks(fs_info, &allocation, data_profile, metadata_profile, metadata_profile); if (ret < 0) { - error("failed to cleanup temporary chunks: %d", ret); + errno = -ret; + error("failed to cleanup temporary chunks: %m"); goto out; } @@ -2111,7 +2136,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv) features.incompat_flags & BTRFS_FEATURE_INCOMPAT_SIMPLE_QUOTA) { ret = setup_quota_root(fs_info); if (ret < 0) { - error("failed to initialize quota: %d (%m)", ret); + errno = -ret; + error("failed to initialize quota: %m"); goto out; } } @@ -2184,8 +2210,8 @@ int BOX_MAIN(mkfs)(int argc, char **argv) if (!ret && close_ret) { ret = close_ret; - error("failed to close ctree, the filesystem may be inconsistent: %d", - ret); + errno = -ret; + error("failed to close ctree, filesystem may be inconsistent: %m"); } btrfs_close_all_devices(); diff --git a/tests/README.md b/tests/README.md index 8e5d59a5fb..1bd0e7c3f9 100644 --- a/tests/README.md +++ b/tests/README.md @@ -8,6 +8,12 @@ management. In some cases optional features are turned on by mkfs and the filesystem image could be mounted, such tests might fail if there's lack of support. +The testsuite follows the principles: + +- gather all output to logs +- stop on first failure (do not clean up until the problem is investigated) +- detect available capabilities in the system, skip if missing + ## Quick start Run the tests from the top directory: @@ -199,7 +205,7 @@ command `losetup` and eventually delete all existing loop devices with `losetup ### Prototyping tests, quick tests There's a script `test-console.sh` that will run shell commands in a loop and -logs the output with the testing environment set up. It sources the common +log the output with the testing environment set up. It sources the common helper scripts so the shell functions are available. ### Runtime dependencies @@ -213,7 +219,8 @@ specific tests need the following packages installed: `acl`, `attr`, 1. Pick the category for the new test or fallback to `misc-tests` if not sure. For an easy start copy an existing `test.sh` script from some test that might be -close to the purpose of your new test. The environment setup includes the +close to the purpose of your new test, or use one of the templates in the +[tests/template/](template/) directory. The environment setup includes the common scripts and/or prepares the test devices. Other scripts contain examples how to do mkfs, mount, unmount, check, loop device management etc. @@ -229,22 +236,23 @@ infrastructure. 5. **Test your test.** Use the `TEST` variable to jump right to your test: ```shell -$ make TEST=012\* tests-misc # from top directory -$ TEST=012\* ./misc-tests.sh # from tests/ +$ make TEST=012\* tests-misc # from the top or tests/ directory +$ TEST=012\* ./misc-tests.sh # alternatively from tests/ ``` 6. The commit changelog should reference a commit that either introduced or - fixed the bug (or both). Subject line of the shall mention the name of the - new directory for ease of search, e.g. `btrfs-progs: tests: add 012-subvolume-sync-must-wait` + fixed the bug (or both). Subject line of the could mention the name of the + new directory for ease of search (e.g. `btrfs-progs: tests: add 012-subvolume-sync-must-wait`) + or a brief description of the test case (e.g. `btrfs-progs: tests: add case for waiting on subvolume sync after deletion`) -7. A commit that fixes a bug should be applied before the test that verifies +7. A commit that fixes a bug should be applied *before* the test that verifies the fix. This is to keep the git history bisectable. ### Test images Most tests should be able to create the test images from scratch, using regular -commands and file operation. The commands also document the test case and use +commands and file operations. The commands also document the test case and use the test code and kernel of the environment. In other cases, a pre-created image may be the right way if the above does not @@ -295,7 +303,8 @@ There are some utilities that are not distributed but are necessary for the tests. They are in the top level directory of the testsuite and their path cannot be set. -The tests assume write access to their directories. +The tests assume write access to their directories and an existing `/tmp` +directory. # Coding style, best practices @@ -305,9 +314,10 @@ The tests assume write access to their directories. * quote all variables by default, any path, even the TOP could need that, and we use it everywhere * even if the variable is safe, use quotes for consistency and to ease - reading the code + reading the code (syntax highlighting in editors) * there are exceptions: * `$SUDO_HELPER` as it might be intentionally unset + * conditionally set options that are stored in a variable * use `#!/bin/bash` explicitly * check for all external dependencies (`check_global_prereq`) * check for internal dependencies (`check_prereq`), though the basic set is diff --git a/tests/convert-tests/027-large-symbol-link/test.sh b/tests/convert-tests/027-large-symbolic-link/test.sh similarity index 60% rename from tests/convert-tests/027-large-symbol-link/test.sh rename to tests/convert-tests/027-large-symbolic-link/test.sh index 2a001424df..e88f2939aa 100755 --- a/tests/convert-tests/027-large-symbol-link/test.sh +++ b/tests/convert-tests/027-large-symbolic-link/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Make sure btrfs-convert can handle a symbol link which is 4095 bytes large +# Make sure btrfs-convert can handle a symbolic link which is 4095 bytes long source "$TEST_TOP/common" || exit source "$TEST_TOP/common.convert" || exit @@ -12,16 +12,16 @@ check_global_prereq mkfs.ext4 link_target=$(printf "%0.sb" {1..4095}) convert_test_prep_fs ext4 mke2fs -t ext4 -b 4096 -run_check $SUDO_HELPER ln -s "$link_target" "$TEST_MNT/symbol_link" +run_check $SUDO_HELPER ln -s "$link_target" "$TEST_MNT/symbolic_link" run_check_umount_test_dev # For unpatched btrfs-convert, it will always append one byte to the -# link target, causing above 4095 target to be 4096, exactly one sector, -# resulting a regular file extent. +# link target, causing the above 4095 target to be 4096, exactly one sector, +# resulting in a regular file extent. convert_test_do_convert run_check_mount_test_dev # If the unpatched btrfs-convert created a regular extent, and the kernel is -# newer enough, such readlink will be rejected by kernel. -run_check $SUDO_HELPER readlink "$TEST_MNT/symbol_link" +# new enough, readlink will be rejected by kernel. +run_check $SUDO_HELPER readlink "$TEST_MNT/symbolic_link" run_check_umount_test_dev diff --git a/tests/misc-tests/064-reverse-receive/test.sh b/tests/misc-tests/061-reverse-receive/test.sh similarity index 100% rename from tests/misc-tests/064-reverse-receive/test.sh rename to tests/misc-tests/061-reverse-receive/test.sh diff --git a/tests/misc-tests/065-image-filename/test.sh b/tests/misc-tests/066-image-filename/test.sh similarity index 100% rename from tests/misc-tests/065-image-filename/test.sh rename to tests/misc-tests/066-image-filename/test.sh diff --git a/tests/misc-tests/065-btrfstune-simple-quota/test.sh b/tests/misc-tests/067-btrfstune-simple-quota/test.sh similarity index 100% rename from tests/misc-tests/065-btrfstune-simple-quota/test.sh rename to tests/misc-tests/067-btrfstune-simple-quota/test.sh diff --git a/tests/template/check-all-images b/tests/template/test.sh.check-all-images old mode 100644 new mode 100755 similarity index 100% rename from tests/template/check-all-images rename to tests/template/test.sh.check-all-images diff --git a/tests/template/multi-mkfs-mount b/tests/template/test.sh.multi-mkfs-mount old mode 100644 new mode 100755 similarity index 100% rename from tests/template/multi-mkfs-mount rename to tests/template/test.sh.multi-mkfs-mount diff --git a/tests/template/single-mkfs-mount b/tests/template/test.sh.single-mkfs-mount old mode 100644 new mode 100755 similarity index 100% rename from tests/template/single-mkfs-mount rename to tests/template/test.sh.single-mkfs-mount