From 963d1b41ddec370c309936b3cb7d29e668a305e4 Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Mon, 16 Dec 2024 22:09:51 -0800 Subject: [PATCH] [test] Checking if criu cgroup v1 kludges help Testing criu PR 2545. Makes TestCheckpoint and TestUsernsCheckpoint run 150 times. For gha ci, use both precompiled criu and compile the one with freezer-kludges. For cirrus/almalinux, only compile criu with freezer-kludges. Disabled some CI runs: - gha: ubuntu 24.04, actuated-arm (cgroup v2); - cirrus: fedora (cgroup v2); - both gha and cirrus: bats installation, integration tests. Signed-off-by: Kir Kolyshkin --- .cirrus.yml | 119 ++------- .github/workflows/test.yml | 89 +------ .github/workflows/validate.yml | 261 -------------------- Makefile | 2 +- libcontainer/integration/checkpoint_test.go | 253 ++++++++++--------- 5 files changed, 161 insertions(+), 563 deletions(-) delete mode 100644 .github/workflows/validate.yml diff --git a/.cirrus.yml b/.cirrus.yml index e63ef51cf1a..d98bd3b5219 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -7,70 +7,6 @@ # NOTE Cirrus execution environments lack a terminal, needed for # some integration tests. So we use `ssh -tt` command to fake a terminal. -task: - timeout_in: 30m - - env: - DEBIAN_FRONTEND: noninteractive - HOME: /root - # yamllint disable rule:key-duplicates - matrix: - DISTRO: fedora - - name: vagrant DISTRO:$DISTRO - - compute_engine_instance: - image_project: cirrus-images - image: family/docker-kvm - platform: linux - nested_virtualization: true - # CPU limit: `16 / NTASK`: see https://cirrus-ci.org/faq/#are-there-any-limits - cpu: 4 - # Memory limit: `4GB * NCPU` - memory: 16G - - host_info_script: | - uname -a - # ----- - cat /etc/os-release - # ----- - df -T - # ----- - cat /proc/cpuinfo - install_libvirt_vagrant_script: | - curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg - echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list - sudo sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list - apt-get update - apt-get install -y libvirt-daemon libvirt-daemon-system vagrant - systemctl enable --now libvirtd - apt-get build-dep -y vagrant ruby-libvirt - apt-get install -y --no-install-recommends libxslt-dev libxml2-dev libvirt-dev ruby-bundler ruby-dev zlib1g-dev - vagrant plugin install vagrant-libvirt - vagrant_cache: - fingerprint_script: cat Vagrantfile.$DISTRO - folder: /root/.vagrant.d/boxes - vagrant_up_script: | - ln -sf Vagrantfile.$DISTRO Vagrantfile - # Retry if it fails (download.fedoraproject.org returns 404 sometimes) - vagrant up --no-tty || vagrant up --no-tty - mkdir -p -m 0700 /root/.ssh - vagrant ssh-config >> /root/.ssh/config - guest_info_script: | - ssh default 'sh -exc "uname -a && systemctl --version && df -T && cat /etc/os-release && go version && sestatus && rpm -q container-selinux"' - check_config_script: | - ssh default /vagrant/script/check-config.sh - unit_tests_script: | - ssh default 'sudo -i make -C /vagrant localunittest' - integration_systemd_script: | - ssh -tt default "sudo -i make -C /vagrant localintegration RUNC_USE_SYSTEMD=yes" - integration_fs_script: | - ssh -tt default "sudo -i make -C /vagrant localintegration" - integration_systemd_rootless_script: | - ssh -tt default "sudo -i make -C /vagrant localrootlessintegration RUNC_USE_SYSTEMD=yes" - integration_fs_rootless_script: | - ssh -tt default "sudo -i make -C /vagrant localrootlessintegration" - task: timeout_in: 30m @@ -79,7 +15,7 @@ task: CIRRUS_WORKING_DIR: /home/runc GO_VERSION: "1.23" BATS_VERSION: "v1.9.0" - RPMS: gcc git iptables jq glibc-static libseccomp-devel make criu fuse-sshfs container-selinux + RPMS: gcc git glibc-static libseccomp-devel make fuse-sshfs container-selinux gnutls-devel libaio-devel libasan libcap-devel libnet-devel libnl3-devel libselinux-devel protobuf-c-devel protobuf-devel libdrm-devel # yamllint disable rule:key-duplicates matrix: DISTRO: almalinux-8 @@ -100,6 +36,7 @@ task: yum config-manager --set-enabled powertools # for glibc-static ;; *-9) + RPMS="$RPMS nftables-devel libbsd-devel" dnf config-manager --set-enabled crb # for glibc-static dnf -y install epel-release # for fuse-sshfs # Delegate all cgroup v2 controllers to rootless user via --systemd-cgroup. @@ -121,21 +58,9 @@ task: # Find out the latest minor release URL. filename=$(curl -fsSL "${PREFIX}?mode=json&include=all" | jq -r --arg Ver "go$GO_VERSION." '. | map(select(.version | contains($Ver))) | first | .files[] | select(.os == "linux" and .arch == "amd64" and .kind == "archive") | .filename') curl -fsSL "$PREFIX$filename" | tar Cxz /usr/local - # install bats - cd /tmp - git clone https://github.com/bats-core/bats-core - cd bats-core - git checkout $BATS_VERSION - ./install.sh /usr/local - cd - - # Add a user for rootless tests - useradd -u2000 -m -d/home/rootless -s/bin/bash rootless - # Allow root and rootless itself to execute `ssh rootless@localhost` in tests/rootless.sh - ssh-keygen -t ecdsa -N "" -f /root/rootless.key - mkdir -m 0700 -p /home/rootless/.ssh - cp /root/rootless.key /home/rootless/.ssh/id_ecdsa - cat /root/rootless.key.pub >> /home/rootless/.ssh/authorized_keys - chown -R rootless.rootless /home/rootless + # Testing https://github.com/checkpoint-restore/criu/pull/2545 + git clone https://github.com/kolyshkin/criu.git ~/criu + (cd ~/criu && git checkout freeze-kludges && sudo make install-criu) # set PATH echo 'export PATH=/usr/local/go/bin:/usr/local/bin:$PATH' >> /root/.bashrc # Setup ssh localhost for terminal emulation (script -e did not work) @@ -146,7 +71,10 @@ task: echo -e "Host localhost\n\tStrictHostKeyChecking no\t\nIdentityFile /root/.ssh/id_ed25519\n" >> /root/.ssh/config sed -e "s,PermitRootLogin.*,PermitRootLogin prohibit-password,g" -i /etc/ssh/sshd_config systemctl restart sshd + rm libcontainer/cgroups/manager/manager_test.go # takes 8s host_info_script: | + criu --version + # ----- uname -a # ----- /usr/local/go/bin/go version @@ -160,21 +88,20 @@ task: sestatus # ----- cat /proc/cpuinfo - check_config_script: | - /home/runc/script/check-config.sh unit_tests_script: | ssh -tt localhost "make -C /home/runc localunittest" - integration_systemd_script: | - ssh -tt localhost "make -C /home/runc localintegration RUNC_USE_SYSTEMD=yes" - integration_fs_script: | - ssh -tt localhost "make -C /home/runc localintegration" - integration_systemd_rootless_script: | - case $DISTRO in - *-8) - echo "SKIP: integration_systemd_rootless_script requires cgroup v2" - ;; - *) - ssh -tt localhost "make -C /home/runc localrootlessintegration RUNC_USE_SYSTEMD=yes" - esac - integration_fs_rootless_script: | - ssh -tt localhost "make -C /home/runc localrootlessintegration" + install_rpm_criu_script: | + mv /usr/local/sbin/criu{,-removed} + yum -y install criu + hash -r + which criu + criu --version + unit_tests_stock_criu_script: | + ssh -tt localhost 'make TESTFLAGS="-count 2" -C /home/runc localunittest' + criu_is_back_script: | + mv /usr/local/sbin/criu{-removed,} + hash -r + which criu + criu --version + unit_tests_again_script: | + ssh -tt localhost 'make TESTFLAGS="-count 2" -C /home/runc localunittest' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 754fd87f155..3ef1dc1abe9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,49 +23,14 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-20.04, ubuntu-24.04, actuated-arm64-6cpu-8gb] + os: [ubuntu-20.04] go-version: [1.22.x, 1.23.x] - rootless: ["rootless", ""] + rootless: [""] race: ["-race", ""] criu: ["", "criu-dev"] - exclude: - # Disable most of criu-dev jobs, as they are expensive - # (need to compile criu) and don't add much value/coverage. - - criu: criu-dev - go-version: 1.22.x - - criu: criu-dev - rootless: rootless - - criu: criu-dev - race: -race - - go-version: 1.22.x - os: actuated-arm64-6cpu-8gb - - race: "-race" - os: actuated-arm64-6cpu-8gb - - criu: criu-dev - os: actuated-arm64-6cpu-8gb - runs-on: ${{ matrix.os }} steps: -# https://gist.github.com/alexellis/1f33e581c75e11e161fe613c46180771#file-metering-gha-md -# vmmeter start - - name: Prepare arkade - uses: alexellis/arkade-get@master - if: matrix.os == 'actuated-arm64-6cpu-8gb' - with: - crane: latest - print-summary: false - - - name: Install vmmeter - if: matrix.os == 'actuated-arm64-6cpu-8gb' - run: | - crane export --platform linux/arm64 ghcr.io/openfaasltd/vmmeter:latest | sudo tar -xvf - -C /usr/local/bin - - - name: Run vmmeter - uses: self-actuated/vmmeter-action@master - if: matrix.os == 'actuated-arm64-6cpu-8gb' -# vmmeter end - - name: checkout uses: actions/checkout@v4 @@ -74,6 +39,8 @@ jobs: set -x # Sync `set -x` outputs with command ouputs exec 2>&1 + # CRIU + criu --version # Version uname -a cat /etc/os-release @@ -124,65 +91,23 @@ jobs: sudo apt -qy install \ libcap-dev libnet1-dev libnl-3-dev \ libprotobuf-c-dev libprotobuf-dev protobuf-c-compiler protobuf-compiler - git clone https://github.com/checkpoint-restore/criu.git ~/criu - (cd ~/criu && git checkout ${{ matrix.criu }} && sudo make install-criu) + # Testing https://github.com/checkpoint-restore/criu/pull/2545 + git clone https://github.com/kolyshkin/criu.git ~/criu + (cd ~/criu && git checkout freeze-kludges && sudo make install-criu) rm -rf ~/criu - name: install go ${{ matrix.go-version }} uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} - check-latest: true - name: build run: sudo -E PATH="$PATH" make EXTRA_FLAGS="${{ matrix.race }}" all - - name: Setup Bats and bats libs - uses: bats-core/bats-action@3.0.0 - with: - bats-version: 1.9.0 - support-install: false - assert-install: false - detik-install: false - file-install: false - - - name: Allow userns for runc - # https://discourse.ubuntu.com/t/ubuntu-24-04-lts-noble-numbat-release-notes/39890#unprivileged-user-namespace-restrictions-15 - if: matrix.os == 'ubuntu-24.04' - run: | - sed "s;^profile runc /usr/sbin/;profile runc-test $PWD/;" < /etc/apparmor.d/runc | sudo apparmor_parser - - name: unit test if: matrix.rootless != 'rootless' run: sudo -E PATH="$PATH" -- make TESTFLAGS="${{ matrix.race }}" localunittest - - name: add rootless user - if: matrix.rootless == 'rootless' - run: | - sudo useradd -u2000 -m -d/home/rootless -s/bin/bash rootless - # Allow root and rootless itself to execute `ssh rootless@localhost` in tests/rootless.sh - ssh-keygen -t ecdsa -N "" -f $HOME/rootless.key - sudo mkdir -m 0700 -p /home/rootless/.ssh - sudo cp $HOME/rootless.key /home/rootless/.ssh/id_ecdsa - sudo cp $HOME/rootless.key.pub /home/rootless/.ssh/authorized_keys - sudo chown -R rootless.rootless /home/rootless - sudo chmod a+X $HOME # for Ubuntu 22.04 and later - - - name: integration test (fs driver) - run: sudo -E PATH="$PATH" script -e -c 'make local${{ matrix.rootless }}integration' - - - name: integration test (systemd driver) - # Skip rootless+systemd for ubuntu 20.04 because of cgroup v1. - if: ${{ !(matrix.os == 'ubuntu-20.04' && matrix.rootless == 'rootless') }} - run: | - # Delegate all cgroup v2 controllers to rootless user via --systemd-cgroup. - # The default (since systemd v252) is "pids memory cpu". - sudo mkdir -p /etc/systemd/system/user@.service.d - printf "[Service]\nDelegate=yes\n" | sudo tee /etc/systemd/system/user@.service.d/delegate.conf - sudo systemctl daemon-reload - # Run the tests. - sudo -E PATH="$PATH" script -e -c 'make RUNC_USE_SYSTEMD=yes local${{ matrix.rootless }}integration' - # We need to continue support for 32-bit ARM. # However, we do not have 32-bit ARM CI, so we use i386 for testing 32bit stuff. # We are not interested in providing official support for i386. diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml deleted file mode 100644 index 3b565f05215..00000000000 --- a/.github/workflows/validate.yml +++ /dev/null @@ -1,261 +0,0 @@ -name: validate -on: - push: - tags: - - v* - branches: - - main - - release-* - pull_request: -env: - GO_VERSION: 1.23.x -permissions: - contents: read - -jobs: - keyring: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - name: check runc.keyring - run: make validate-keyring - - lint: - timeout-minutes: 30 - permissions: - contents: read - pull-requests: read - checks: write # to allow the action to annotate code in the PR. - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - uses: actions/setup-go@v5 - with: - go-version: "${{ env.GO_VERSION }}" - - name: install deps - run: | - sudo apt -q update - sudo apt -qy install libseccomp-dev - - uses: golangci/golangci-lint-action@v6 - with: - version: v1.60 - # Extra linters, only checking new code from a pull request. - - name: lint-extra - if: github.event_name == 'pull_request' - run: | - golangci-lint run --config .golangci-extra.yml --new-from-rev=HEAD~1 - - go-fix: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - uses: actions/setup-go@v5 - with: - go-version: "${{ env.GO_VERSION }}" - - name: install deps - run: | - sudo apt -q update - sudo apt -qy install libseccomp-dev - - name: run go fix - run: | - go fix ./... - git diff --exit-code - - compile-buildtags: - runs-on: ubuntu-24.04 - env: - # Don't ignore C warnings. Note that the output of "go env CGO_CFLAGS" by default is "-g -O2", so we keep them. - CGO_CFLAGS: -g -O2 -Werror - steps: - - uses: actions/checkout@v4 - - name: install go - uses: actions/setup-go@v5 - with: - go-version: "${{ env.GO_VERSION }}" - - name: install deps - run: | - sudo apt update - sudo apt -y install libseccomp-dev - - name: compile with no build tags - run: make BUILDTAGS="" - - name: compile with runc_nocriu build tag - run: make EXTRA_BUILDTAGS="runc_nocriu" - - codespell: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - name: install deps - # Version of codespell bundled with Ubuntu is way old, so use pip. - run: pip install --break-system-packages codespell==v2.3.0 - - name: run codespell - run: codespell - - shfmt: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - name: shfmt - run: make shfmt - - shellcheck: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - name: install shellcheck - env: - VERSION: v0.9.0 - BASEURL: https://github.com/koalaman/shellcheck/releases/download - SHA256: 7087178d54de6652b404c306233264463cb9e7a9afeb259bb663cc4dbfd64149 - run: | - mkdir ~/bin - curl -sSfL --retry 5 $BASEURL/$VERSION/shellcheck-$VERSION.linux.x86_64.tar.xz | - tar xfJ - -C ~/bin --strip 1 shellcheck-$VERSION/shellcheck - sha256sum --strict --check - <<<"$SHA256 *$HOME/bin/shellcheck" - # make sure to remove the old version - sudo rm -f /usr/bin/shellcheck - # Add ~/bin to $PATH. - echo ~/bin >> $GITHUB_PATH - - uses: lumaxis/shellcheck-problem-matchers@v2 - - name: run - run: make shellcheck - - name: check-config.sh - run : ./script/check-config.sh - - space-at-eol: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - run: rm -fr vendor - - run: if git -P grep -I -n '\s$'; then echo "^^^ extra whitespace at EOL, please fix"; exit 1; fi - - deps: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - name: install go - uses: actions/setup-go@v5 - with: - go-version: "${{ env.GO_VERSION }}" - check-latest: true - - name: verify deps - run: make verify-dependencies - - - commit: - permissions: - contents: read - pull-requests: read - runs-on: ubuntu-24.04 - # Only check commits on pull requests. - if: github.event_name == 'pull_request' - steps: - - name: get pr commits - id: 'get-pr-commits' - uses: tim-actions/get-pr-commits@v1.3.1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - name: check subject line length - uses: tim-actions/commit-message-checker-with-regex@v0.3.2 - with: - commits: ${{ steps.get-pr-commits.outputs.commits }} - pattern: '^.{0,72}(\n.*)*$' - error: 'Subject too long (max 72)' - - cfmt: - runs-on: ubuntu-24.04 - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: install deps - run: | - sudo apt -qq update - sudo apt -qqy install indent - - name: cfmt - run: | - make cfmt - git diff --exit-code - - - release: - timeout-minutes: 30 - runs-on: ubuntu-24.04 - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: check CHANGELOG.md - run: make verify-changelog - - # We have to run this under Docker as Ubuntu (host) does not support all - # the architectures we want to compile test against, and Dockerfile uses - # Debian (which does). - # - # XXX: as currently this is the only job that is using Docker, we are - # building and using the runcimage locally. In case more jobs running - # under Docker will emerge, it will be good to have a separate make - # runcimage job and share its result (the docker image) with whoever - # needs it. - - name: build docker image - run: make runcimage - - name: make releaseall - run: make releaseall - - name: upload artifacts - uses: actions/upload-artifact@v4 - with: - name: release-${{ github.run_id }} - path: release/* - - - get-images: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: install bashbrew - env: - BASEURL: https://github.com/docker-library/bashbrew/releases/download - VERSION: v0.1.7 - SHA256: 6b71a6fccfb2025d48a2b23324836b5513c29abfd2d16a57b7a2f89bd02fe53a - run: | - mkdir ~/bin - curl -sSfL --retry 5 -o ~/bin/bashbrew \ - $BASEURL/$VERSION/bashbrew-amd64 - sha256sum --strict --check - <<<"$SHA256 *$HOME/bin/bashbrew" - chmod a+x ~/bin/bashbrew - # Add ~/bin to $PATH. - echo ~/bin >> $GITHUB_PATH - - name: check that get-images.sh is up to date - run: | - cd tests/integration - ./bootstrap-get-images.sh > get-images.sh - git diff --exit-code - - all-done: - needs: - - cfmt - - codespell - - commit - - compile-buildtags - - deps - - get-images - - go-fix - - keyring - - lint - - release - - shellcheck - - shfmt - - space-at-eol - runs-on: ubuntu-24.04 - steps: - - run: echo "All jobs completed" diff --git a/Makefile b/Makefile index 39c1ef918d4..dbe86e47d68 100644 --- a/Makefile +++ b/Makefile @@ -159,7 +159,7 @@ unittest: runcimage .PHONY: localunittest localunittest: test-binaries - $(GO) test -timeout 3m -tags "$(BUILDTAGS)" $(TESTFLAGS) -v ./... + $(GO) test -timeout 10m -tags "$(BUILDTAGS)" $(TESTFLAGS) -v ./... .PHONY: integration integration: runcimage diff --git a/libcontainer/integration/checkpoint_test.go b/libcontainer/integration/checkpoint_test.go index 8d4d6fe4751..03aafd697d9 100644 --- a/libcontainer/integration/checkpoint_test.go +++ b/libcontainer/integration/checkpoint_test.go @@ -6,6 +6,7 @@ import ( "os/exec" "path/filepath" "regexp" + "strconv" "strings" "testing" @@ -17,6 +18,8 @@ func criuFeature(feature string) bool { return exec.Command("criu", "check", "--feature", feature).Run() == nil } +const iter = 150 + func TestUsernsCheckpoint(t *testing.T) { testCheckpoint(t, true) } @@ -44,128 +47,132 @@ func testCheckpoint(t *testing.T, userns bool) { t.Skip("Test requires userns") } - config := newTemplateConfig(t, &tParam{userns: userns}) - stateDir := t.TempDir() - - container, err := libcontainer.Create(stateDir, "test", config) - ok(t, err) - defer destroyContainer(container) - - stdinR, stdinW, err := os.Pipe() - ok(t, err) - - var stdout bytes.Buffer - - pconfig := libcontainer.Process{ - Cwd: "/", - Args: []string{"cat"}, - Env: standardEnvironment, - Stdin: stdinR, - Stdout: &stdout, - Init: true, - } - - err = container.Run(&pconfig) - _ = stdinR.Close() - defer stdinW.Close() //nolint: errcheck - ok(t, err) - - pid, err := pconfig.Pid() - ok(t, err) - - process, err := os.FindProcess(pid) - ok(t, err) - - tmp := t.TempDir() - var parentImage string - - // Test pre-dump if mem_dirty_track is available. - if criuFeature("mem_dirty_track") { - parentImage = "../criu-parent" - parentDir := filepath.Join(tmp, "criu-parent") - preDumpOpts := &libcontainer.CriuOpts{ - ImagesDirectory: parentDir, - WorkDirectory: parentDir, - PreDump: true, - } - - if err := container.Checkpoint(preDumpOpts); err != nil { - t.Fatal(err) - } - - state, err := container.Status() - ok(t, err) - - if state != libcontainer.Running { - t.Fatal("Unexpected preDump state: ", state) - } - } - - imagesDir := filepath.Join(tmp, "criu") - - checkpointOpts := &libcontainer.CriuOpts{ - ImagesDirectory: imagesDir, - WorkDirectory: imagesDir, - ParentImage: parentImage, - } - - if err := container.Checkpoint(checkpointOpts); err != nil { - t.Fatal(err) - } - - state, err := container.Status() - ok(t, err) - - if state != libcontainer.Stopped { - t.Fatal("Unexpected state checkpoint: ", state) - } - - _ = stdinW.Close() - _, err = process.Wait() - ok(t, err) - - // reload the container - container, err = libcontainer.Load(stateDir, "test") - ok(t, err) - - restoreStdinR, restoreStdinW, err := os.Pipe() - ok(t, err) - - var restoreStdout bytes.Buffer - restoreProcessConfig := &libcontainer.Process{ - Cwd: "/", - Stdin: restoreStdinR, - Stdout: &restoreStdout, - Init: true, - } - - err = container.Restore(restoreProcessConfig, checkpointOpts) - _ = restoreStdinR.Close() - defer restoreStdinW.Close() //nolint: errcheck - if err != nil { - t.Fatal(err) - } - - state, err = container.Status() - ok(t, err) - if state != libcontainer.Running { - t.Fatal("Unexpected restore state: ", state) - } - - pid, err = restoreProcessConfig.Pid() - ok(t, err) - - err = unix.Kill(pid, 0) - ok(t, err) - - _, err = restoreStdinW.WriteString("Hello!") - ok(t, err) - - _ = restoreStdinW.Close() - waitProcess(restoreProcessConfig, t) - - output := restoreStdout.String() - if !strings.Contains(output, "Hello!") { - t.Fatal("Did not restore the pipe correctly:", output) + for i := range iter { + t.Run(strconv.Itoa(i), func(t *testing.T) { + config := newTemplateConfig(t, &tParam{userns: userns}) + stateDir := t.TempDir() + + container, err := libcontainer.Create(stateDir, "test", config) + ok(t, err) + defer destroyContainer(container) + + stdinR, stdinW, err := os.Pipe() + ok(t, err) + + var stdout bytes.Buffer + + pconfig := libcontainer.Process{ + Cwd: "/", + Args: []string{"cat"}, + Env: standardEnvironment, + Stdin: stdinR, + Stdout: &stdout, + Init: true, + } + + err = container.Run(&pconfig) + _ = stdinR.Close() + defer stdinW.Close() //nolint: errcheck + ok(t, err) + + pid, err := pconfig.Pid() + ok(t, err) + + process, err := os.FindProcess(pid) + ok(t, err) + + tmp := t.TempDir() + var parentImage string + + // Test pre-dump if mem_dirty_track is available. + if criuFeature("mem_dirty_track") { + parentImage = "../criu-parent" + parentDir := filepath.Join(tmp, "criu-parent") + preDumpOpts := &libcontainer.CriuOpts{ + ImagesDirectory: parentDir, + WorkDirectory: parentDir, + PreDump: true, + } + + if err := container.Checkpoint(preDumpOpts); err != nil { + t.Fatal(err) + } + + state, err := container.Status() + ok(t, err) + + if state != libcontainer.Running { + t.Fatal("Unexpected preDump state: ", state) + } + } + + imagesDir := filepath.Join(tmp, "criu") + + checkpointOpts := &libcontainer.CriuOpts{ + ImagesDirectory: imagesDir, + WorkDirectory: imagesDir, + ParentImage: parentImage, + } + + if err := container.Checkpoint(checkpointOpts); err != nil { + t.Fatal(err) + } + + state, err := container.Status() + ok(t, err) + + if state != libcontainer.Stopped { + t.Fatal("Unexpected state checkpoint: ", state) + } + + _ = stdinW.Close() + _, err = process.Wait() + ok(t, err) + + // reload the container + container, err = libcontainer.Load(stateDir, "test") + ok(t, err) + + restoreStdinR, restoreStdinW, err := os.Pipe() + ok(t, err) + + var restoreStdout bytes.Buffer + restoreProcessConfig := &libcontainer.Process{ + Cwd: "/", + Stdin: restoreStdinR, + Stdout: &restoreStdout, + Init: true, + } + + err = container.Restore(restoreProcessConfig, checkpointOpts) + _ = restoreStdinR.Close() + defer restoreStdinW.Close() //nolint: errcheck + if err != nil { + t.Fatal(err) + } + + state, err = container.Status() + ok(t, err) + if state != libcontainer.Running { + t.Fatal("Unexpected restore state: ", state) + } + + pid, err = restoreProcessConfig.Pid() + ok(t, err) + + err = unix.Kill(pid, 0) + ok(t, err) + + _, err = restoreStdinW.WriteString("Hello!") + ok(t, err) + + _ = restoreStdinW.Close() + waitProcess(restoreProcessConfig, t) + + output := restoreStdout.String() + if !strings.Contains(output, "Hello!") { + t.Fatal("Did not restore the pipe correctly:", output) + } + }) } }