merge 'windows' branch from upstream

.github/ISSUE_TEMPLATE/bug_report.md

@@ -2,7 +2,7 @@
 name: Bug report
 about: Create a report to help us improve OpenZFS
 title: ''
-labels: 'Type: Defect, Status: Triage Needed'
+labels: 'Type: Defect'
 assignees: ''
 
 ---
@@ -25,14 +25,16 @@ Type | Version/Name
  --- | ---
 Distribution Name	|
 Distribution Version	|
-Linux Kernel	|
+Kernel Version	|
 Architecture	|
-ZFS Version	|
-SPL Version	|
+OpenZFS Version	|
 <!--
-Commands to find ZFS/SPL versions:
-modinfo zfs | grep -iw version
-modinfo spl | grep -iw version
+Command to find OpenZFS version:
+zfs version
+
+Commands to find kernel version:
+uname -r		# Linux
+freebsd-version -r	# FreeBSD
 -->
 
 ### Describe the problem you're observing

.github/workflows/build-dependencies.txt

@@ -0,0 +1,48 @@
+acl
+alien
+attr
+autoconf
+bc
+build-essential
+curl
+dbench
+fakeroot
+fio
+gdb
+gdebi
+git
+ksh
+lcov
+libacl1-dev
+libaio-dev
+libattr1-dev
+libblkid-dev
+libcurl4-openssl-dev
+libdevmapper-dev
+libelf-dev
+libffi-dev
+libmount-dev
+libpam0g-dev
+libselinux-dev
+libssl-dev
+libtool
+libudev-dev
+lsscsi
+mdadm
+nfs-kernel-server
+pamtester
+parted
+python3
+python3-cffi
+python3-dev
+python3-packaging
+python3-setuptools
+rng-tools
+samba
+sysstat
+uuid-dev
+watchdog
+wget
+xfslibs-dev
+xz-utils
+zlib1g-dev

.github/workflows/checkstyle.yaml

@@ -6,31 +6,51 @@ on:
 
 jobs:
   checkstyle:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
       with:
         ref: ${{ github.event.pull_request.head.sha }}
     - name: Install dependencies
       run: |
         sudo apt-get update
-        sudo apt-get install --yes -qq build-essential autoconf libtool gawk alien fakeroot linux-headers-$(uname -r)
-        sudo apt-get install --yes -qq zlib1g-dev uuid-dev libattr1-dev libblkid-dev libselinux-dev libudev-dev libssl-dev python-dev python-setuptools python-cffi python3 python3-dev python3-setuptools python3-cffi
-        # packages for tests
-        sudo apt-get install --yes -qq parted lsscsi ksh attr acl nfs-kernel-server fio
-        sudo apt-get install --yes -qq mandoc cppcheck pax-utils devscripts abigail-tools
-        sudo -E pip --quiet install flake8
+        sudo xargs --arg-file=${{ github.workspace }}/.github/workflows/build-dependencies.txt apt-get install -qq
+        sudo apt-get install -qq mandoc cppcheck pax-utils devscripts
+        sudo python3 -m pip install --quiet flake8
+        sudo apt-get clean
+
+        # confirm that the tools are installed
+        # the build system doesn't fail when they are not
+        checkbashisms --version
+        cppcheck --version
+        flake8 --version
+        scanelf --version
+        shellcheck --version
     - name: Prepare
       run: |
-        sh ./autogen.sh
+        ./autogen.sh
         ./configure
-        make -j$(nproc)
+        make -j$(nproc) --no-print-directory --silent
     - name: Checkstyle
       run: |
-        make checkstyle
+        make -j$(nproc) --no-print-directory --silent checkstyle
     - name: Lint
       run: |
-        make lint
+        make -j$(nproc) --no-print-directory --silent lint
     - name: CheckABI
+      id: CheckABI
+      run: |
+        docker run -v $(pwd):/source ghcr.io/openzfs/libabigail make -j$(nproc) --no-print-directory --silent checkabi
+    - name: StoreABI
+      if: failure() && steps.CheckABI.outcome == 'failure'
       run: |
-        make checkabi
+        docker run -v $(pwd):/source ghcr.io/openzfs/libabigail make -j$(nproc) --no-print-directory --silent storeabi
+    - name: Prepare artifacts
+      if: failure() && steps.CheckABI.outcome == 'failure'
+      run: |
+        find -name *.abi | tar -cf abi_files.tar -T -
+    - uses: actions/upload-artifact@v2
+      if: failure() && steps.CheckABI.outcome == 'failure'
+      with:
+        name: New ABI files (use only if you're sure about interface changes)
+        path: abi_files.tar

.github/workflows/scripts/reclaim_disk_space.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -eu
+
+# remove 4GiB of images
+sudo systemd-run docker system prune --force --all --volumes
+
+# remove unused software
+sudo systemd-run rm -rf \
+  "$AGENT_TOOLSDIRECTORY" \
+  /opt/* \
+  /usr/local/* \
+  /usr/share/az* \
+  /usr/share/dotnet \
+  /usr/share/gradle* \
+  /usr/share/miniconda \
+  /usr/share/swift \
+  /var/lib/gems \
+  /var/lib/mysql \
+  /var/lib/snapd

.github/workflows/zfs-tests-functional.yml

@@ -15,28 +15,23 @@ jobs:
     - uses: actions/checkout@v2
       with:
         ref: ${{ github.event.pull_request.head.sha }}
+    - name: Reclaim disk space
+      run: |
+        ${{ github.workspace }}/.github/workflows/scripts/reclaim_disk_space.sh
     - name: Install dependencies
       run: |
         sudo apt-get update
-        sudo apt-get install --yes -qq build-essential autoconf libtool gdb lcov \
-          git alien fakeroot wget curl bc fio acl \
-          sysstat mdadm lsscsi parted gdebi attr dbench watchdog ksh \
-          nfs-kernel-server samba rng-tools xz-utils \
-          zlib1g-dev uuid-dev libblkid-dev libselinux-dev \
-          xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \
-          libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \
-          libpam0g-dev pamtester python-dev python-setuptools python-cffi \
-          python-packaging python3 python3-dev python3-setuptools python3-cffi \
-          libcurl4-openssl-dev python3-packaging
+        xargs --arg-file=${{ github.workspace }}/.github/workflows/build-dependencies.txt sudo apt-get install -qq
+        sudo apt-get clean
     - name: Autogen.sh
       run: |
-        sh autogen.sh
+        ./autogen.sh
     - name: Configure
       run: |
-        ./configure --enable-debug --enable-debuginfo
+        ./configure --enable-debug --enable-debuginfo --enable-asan --enable-ubsan
     - name: Make
       run: |
-        make --no-print-directory -s pkg-utils pkg-kmod
+        make -j$(nproc) --no-print-directory --silent pkg-utils pkg-kmod
     - name: Install
       run: |
         sudo dpkg -i *.deb
@@ -45,27 +40,41 @@ jobs:
         sudo sed -i.bak 's/updates/extra updates/' /etc/depmod.d/ubuntu.conf
         sudo depmod
         sudo modprobe zfs
-        # Workaround to provide additional free space for testing.
-        #   https://github.com/actions/virtual-environments/issues/2840
-        sudo rm -rf /usr/share/dotnet
-        sudo rm -rf /opt/ghc
-        sudo rm -rf "/usr/local/share/boost"
-        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+        # Workaround for cloud-init bug
+        # see https://github.com/openzfs/zfs/issues/12644
+        FILE=/lib/udev/rules.d/10-cloud-init-hook-hotplug.rules
+        if [ -r "${FILE}" ]; then
+          HASH=$(md5sum "${FILE}" | awk '{ print $1 }')
+          if [ "${HASH}" = "121ff0ef1936cd2ef65aec0458a35772" ]; then
+            # Just shove a zd* exclusion right above the hotplug hook...
+            sudo sed -i -e s/'LABEL="cloudinit_hook"'/'KERNEL=="zd*", GOTO="cloudinit_end"\n&'/ "${FILE}"
+            sudo udevadm control --reload-rules
+          fi
+        fi
+    - name: Clear the kernel ring buffer
+      run: |
+        sudo dmesg -c >/var/tmp/dmesg-prerun
+    - name: Report disk space
+      run: |
+        df -h /
     - name: Tests
       run: |
-        /usr/share/zfs/zfs-tests.sh -v -s 3G
+        /usr/share/zfs/zfs-tests.sh -vR -s 3G
+      timeout-minutes: 330
     - name: Prepare artifacts
       if: failure()
       run: |
         RESULTS_PATH=$(readlink -f /var/tmp/test_results/current)
         sudo dmesg > $RESULTS_PATH/dmesg
-        sudo cp /var/log/syslog $RESULTS_PATH/
+        sudo cp /var/log/syslog /var/tmp/dmesg-prerun $RESULTS_PATH/
         sudo chmod +r $RESULTS_PATH/*
         # Replace ':' in dir names, actions/upload-artifact doesn't support it
-        for f in $(find $RESULTS_PATH -name '*:*'); do mv "$f" "${f//:/__}"; done
+        for f in $(find /var/tmp/test_results -name '*:*'); do mv "$f" "${f//:/__}"; done
     - uses: actions/upload-artifact@v2
       if: failure()
       with:
         name: Test logs Ubuntu-${{ matrix.os }}
-        path: /var/tmp/test_results/20*/
+        path: |
+          /var/tmp/test_results/*
+          !/var/tmp/test_results/current
         if-no-files-found: ignore

.github/workflows/zfs-tests-sanity.yml

@@ -6,33 +6,28 @@ on:
 
 jobs:
   tests:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v2
       with:
         ref: ${{ github.event.pull_request.head.sha }}
+    - name: Reclaim disk space
+      run: |
+        ${{ github.workspace }}/.github/workflows/scripts/reclaim_disk_space.sh
     - name: Install dependencies
       run: |
         sudo apt-get update
-        sudo apt-get install --yes -qq build-essential autoconf libtool gdb lcov \
-          git alien fakeroot wget curl bc fio acl \
-          sysstat mdadm lsscsi parted gdebi attr dbench watchdog ksh \
-          nfs-kernel-server samba rng-tools xz-utils \
-          zlib1g-dev uuid-dev libblkid-dev libselinux-dev \
-          xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \
-          libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \
-          libpam0g-dev pamtester python-dev python-setuptools python-cffi \
-          python-packaging python3 python3-dev python3-setuptools python3-cffi \
-          python3-packaging libcurl4-openssl-dev
+        xargs --arg-file=${{ github.workspace }}/.github/workflows/build-dependencies.txt sudo apt-get install -qq
+        sudo apt-get clean
     - name: Autogen.sh
       run: |
-        sh autogen.sh
+        ./autogen.sh
     - name: Configure
       run: |
-        ./configure --enable-debug --enable-debuginfo
+        ./configure --enable-debug --enable-debuginfo --enable-asan --enable-ubsan
     - name: Make
       run: |
-        make --no-print-directory -s pkg-utils pkg-kmod
+        make -j$(nproc) --no-print-directory --silent pkg-utils pkg-kmod
     - name: Install
       run: |
         sudo dpkg -i *.deb
@@ -41,27 +36,41 @@ jobs:
         sudo sed -i.bak 's/updates/extra updates/' /etc/depmod.d/ubuntu.conf
         sudo depmod
         sudo modprobe zfs
-        # Workaround to provide additional free space for testing.
-        #   https://github.com/actions/virtual-environments/issues/2840
-        sudo rm -rf /usr/share/dotnet
-        sudo rm -rf /opt/ghc
-        sudo rm -rf "/usr/local/share/boost"
-        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+        # Workaround for cloud-init bug
+        # see https://github.com/openzfs/zfs/issues/12644
+        FILE=/lib/udev/rules.d/10-cloud-init-hook-hotplug.rules
+        if [ -r "${FILE}" ]; then
+          HASH=$(md5sum "${FILE}" | awk '{ print $1 }')
+          if [ "${HASH}" = "121ff0ef1936cd2ef65aec0458a35772" ]; then
+            # Just shove a zd* exclusion right above the hotplug hook...
+            sudo sed -i -e s/'LABEL="cloudinit_hook"'/'KERNEL=="zd*", GOTO="cloudinit_end"\n&'/ "${FILE}"
+            sudo udevadm control --reload-rules
+          fi
+        fi
+    - name: Clear the kernel ring buffer
+      run: |
+        sudo dmesg -c >/var/tmp/dmesg-prerun
+    - name: Report disk space
+      run: |
+        df -h /
     - name: Tests
       run: |
-        /usr/share/zfs/zfs-tests.sh -v -s 3G -r sanity
+        /usr/share/zfs/zfs-tests.sh -vR -s 3G -r sanity
+      timeout-minutes: 330
     - name: Prepare artifacts
       if: failure()
       run: |
         RESULTS_PATH=$(readlink -f /var/tmp/test_results/current)
         sudo dmesg > $RESULTS_PATH/dmesg
-        sudo cp /var/log/syslog $RESULTS_PATH/
+        sudo cp /var/log/syslog /var/tmp/dmesg-prerun $RESULTS_PATH/
         sudo chmod +r $RESULTS_PATH/*
         # Replace ':' in dir names, actions/upload-artifact doesn't support it
-        for f in $(find $RESULTS_PATH -name '*:*'); do mv "$f" "${f//:/__}"; done
+        for f in $(find /var/tmp/test_results -name '*:*'); do mv "$f" "${f//:/__}"; done
     - uses: actions/upload-artifact@v2
       if: failure()
       with:
-        name: Test logs
-        path: /var/tmp/test_results/20*/
+        name: Test logs Ubuntu-${{ matrix.os }}
+        path: |
+          /var/tmp/test_results/*
+          !/var/tmp/test_results/current
         if-no-files-found: ignore

.github/workflows/zloop.yml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   tests:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     env:
       TEST_DIR: /var/tmp/zloop
     steps:
@@ -16,23 +16,17 @@ jobs:
     - name: Install dependencies
       run: |
         sudo apt-get update
-        sudo apt-get install --yes -qq build-essential autoconf libtool gdb \
-          git alien fakeroot \
-          zlib1g-dev uuid-dev libblkid-dev libselinux-dev \
-          xfslibs-dev libattr1-dev libacl1-dev libudev-dev libdevmapper-dev \
-          libssl-dev libffi-dev libaio-dev libelf-dev libmount-dev \
-          libpam0g-dev \
-          python-dev python-setuptools python-cffi python-packaging \
-          python3 python3-dev python3-setuptools python3-cffi python3-packaging
+        xargs --arg-file=${{ github.workspace }}/.github/workflows/build-dependencies.txt sudo apt-get install -qq
+        sudo apt-get clean
     - name: Autogen.sh
       run: |
-        sh autogen.sh
+        ./autogen.sh
     - name: Configure
       run: |
-        ./configure --enable-debug --enable-debuginfo
+        ./configure --enable-debug --enable-debuginfo --enable-asan --enable-ubsan
     - name: Make
       run: |
-        make --no-print-directory -s pkg-utils pkg-kmod
+        make -j$(nproc) --no-print-directory --silent pkg-utils pkg-kmod
     - name: Install
       run: |
         sudo dpkg -i *.deb
@@ -45,7 +39,7 @@ jobs:
       run: |
         sudo mkdir -p $TEST_DIR
         # run for 20 minutes to have a total runner time of 30 minutes
-        sudo /usr/share/zfs/zloop.sh -t 1200 -l -m1
+        sudo /usr/share/zfs/zloop.sh -t 1200 -l -m1 -- -T 120 -P 60
     - name: Prepare artifacts
       if: failure()
       run: |