[PH] Update to test harness python package

.cicd/platforms/ubuntu18.Dockerfile

@@ -7,13 +7,10 @@ RUN apt-get update && apt-get upgrade -y && \
                                                       g++-8                       \
                                                       git                         \
                                                       jq                          \
-                                                      libcurl4-openssl-dev        \
                                                       libgmp-dev                  \
                                                       libssl-dev                  \
-                                                      libusb-1.0-0-dev            \
                                                       llvm-7-dev                  \
                                                       ninja-build                 \
-                                                      pkg-config                  \
                                                       python3                     \
                                                       software-properties-common  \
                                                       zlib1g-dev                  \

.cicd/platforms/ubuntu20.Dockerfile

@@ -7,11 +7,8 @@ RUN apt-get update && apt-get upgrade -y && \
                                                       git                  \
                                                       jq                   \
                                                       libboost-all-dev     \
-                                                      libcurl4-openssl-dev \
                                                       libgmp-dev           \
                                                       libssl-dev           \
-                                                      libusb-1.0-0-dev     \
                                                       llvm-11-dev          \
                                                       ninja-build          \
-                                                      pkg-config           \
                                                       zstd

.cicd/platforms/ubuntu22.Dockerfile

@@ -7,11 +7,8 @@ RUN apt-get update && apt-get upgrade -y && \
                                                       git                  \
                                                       jq                   \
                                                       libboost-all-dev     \
-                                                      libcurl4-openssl-dev \
                                                       libgmp-dev           \
                                                       libssl-dev           \
-                                                      libusb-1.0-0-dev     \
                                                       llvm-11-dev          \
                                                       ninja-build          \
-                                                      pkg-config           \
                                                       zstd

.github/actions/parallel-ctest-containers/action.yml

@@ -0,0 +1,14 @@
+name: 'Parallel ctest'
+description: 'Runs a set of ctests in parallel via multiple docker containers'
+inputs:
+  container:
+    required: true
+  error-log-paths:
+    required: true
+  log-tarball-prefix:
+    required: true
+  tests:
+    required: true
+runs:
+  using: 'node16'
+  main: 'dist/index.mjs'

.github/actions/parallel-ctest-containers/main.mjs

@@ -0,0 +1,63 @@
+import child_process from 'node:child_process';
+import process from 'node:process';
+import stream from 'node:stream';
+import fs from 'node:fs';
+import zlib from 'node:zlib';
+import tar from 'tar-stream';
+import core from '@actions/core'
+
+const container = core.getInput('container', {required: true});
+const error_log_paths = JSON.parse(core.getInput('error-log-paths', {required: true}));
+const log_tarball_prefix = core.getInput('log-tarball-prefix', {required: true});
+const tests = JSON.parse(core.getInput('tests', {required: true}));
+
+try {
+   if(child_process.spawnSync("docker", ["run", "--name", "base", "-v", `${process.cwd()}/build.tar.zst:/build.tar.zst`, "--workdir", "/__w/leap/leap", container, "tar",  "--zstd", "-xf", "/build.tar.zst"], {stdio:"inherit"}).status)
+      throw new Error("Failed to create base container");
+   if(child_process.spawnSync("docker", ["commit", "base", "baseimage"], {stdio:"inherit"}).status)
+      throw new Error("Failed to create base image");
+   if(child_process.spawnSync("docker", ["rm", "base"], {stdio:"inherit"}).status)
+      throw new Error("Failed to remove base container");
+
+   let subprocesses = [];
+   tests.forEach(t => {
+      subprocesses.push(new Promise(resolve => {
+         child_process.spawn("docker", ["run", "--name", t, "--init", "baseimage", "bash", "-c", `cd build; ctest --output-on-failure -R '^${t}$'`], {stdio:"inherit"}).on('close', code => resolve(code));
+      }));
+   });
+
+   const results = await Promise.all(subprocesses);
+
+   for(let i = 0; i < results.length; ++i) {
+      if(results[i] === 0)
+         continue;
+
+      //failing test
+      core.setFailed("Some tests failed");
+
+      let extractor = tar.extract();
+      let packer = tar.pack();
+
+      extractor.on('entry', (header, stream, next) => {
+         if(!header.name.startsWith(`__w/leap/leap/build`)) {
+            stream.on('end', () => next());
+            stream.resume();
+            return;
+         }
+
+         header.name = header.name.substring(`__w/leap/leap/`.length);
+         if(header.name !== "build/" && error_log_paths.filter(p => header.name.startsWith(p)).length === 0) {
+            stream.on('end', () => next());
+            stream.resume();
+            return;
+         }
+
+         stream.pipe(packer.entry(header, next));
+      }).on('finish', () => {packer.finalize()});
+
+      child_process.spawn("docker", ["export", tests[i]]).stdout.pipe(extractor);
+      stream.promises.pipeline(packer, zlib.createGzip(), fs.createWriteStream(`${log_tarball_prefix}-${tests[i]}-logs.tar.gz`));
+   }
+} catch(e) {
+   core.setFailed(`Uncaught exception ${e.message}`);
+}

.github/actions/parallel-ctest-containers/package.json

@@ -0,0 +1,7 @@
+{
+  "dependencies": {
+    "@actions/core": "^1.9.1",
+    "tar-stream": "^2.2.0"
+  }
+
+}

.github/workflows/build.yaml

@@ -57,6 +57,7 @@ jobs:
           push: true
           tags: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
           file: ${{fromJSON(needs.d.outputs.p)[matrix.platform].dockerfile}}
+
   Build:
     needs: [d, build-platforms]
     if: always() && needs.d.result == 'success' && (needs.build-platforms.result == 'success' || needs.build-platforms.result == 'skipped')
@@ -65,6 +66,7 @@ jobs:
       matrix:
         platform: [ubuntu18, ubuntu20, ubuntu22]
     outputs:
+      np-tests: ${{steps.build.outputs.np-tests}}
       lr-tests: ${{steps.build.outputs.lr-tests}}
     runs-on: ["self-hosted", "enf-x86-beefy"]
     container: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
@@ -82,24 +84,40 @@ jobs:
             cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -GNinja ..
             ninja
             # the correct approach is by far "--show-only=json-v1 | jq '.tests[].name' | jq -sc"  but that doesn't work on U18's cmake 3.10 since it lacks json-v1
+            echo ::set-output name=np-tests::$(ctest -L "nonparallelizable_tests" --show-only | head -n -1 | cut -d ':' -f 2 -s | jq -cnR '[inputs | select(length>0)[1:]]')
             echo ::set-output name=lr-tests::$(ctest -L "long_running_tests" --show-only | head -n -1 | cut -d ':' -f 2 -s | jq -cnR '[inputs | select(length>0)[1:]]')
-            tar -pc -C .. --exclude "*.a" --exclude "*.o" build | zstd --long -T0 -9 > ../build.tar.gz
+            tar -pc -C .. --exclude "*.o" build | zstd --long -T0 -9 > ../build.tar.zst
         - name: Upload builddir
-          uses: actions/upload-artifact@v3
+          uses: AntelopeIO/upload-artifact-large-chunks-action@v1
           with:
             name: ${{matrix.platform}}-build
-            path: build.tar.gz
-        - name: Build dev package (Ubuntu 20 only)
-          if: matrix.platform == 'ubuntu20'
-          run: |
-            cd build
-            ninja package
-        - name: Upload dev package (Ubuntu 20 only)
-          if: matrix.platform == 'ubuntu20'
-          uses: actions/upload-artifact@v3
-          with:
-            name: leap-dev-ubuntu20-amd64
-            path: build/leap-dev*.deb
+            path: build.tar.zst
+
+  dev-package:
+    name: Build leap-dev package
+    needs: [d, Build]
+    if: always() && needs.Build.result == 'success'
+    runs-on: ubuntu-latest
+    container: ${{fromJSON(needs.d.outputs.p)['ubuntu20'].image}}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Download builddir
+        uses: actions/download-artifact@v3
+        with:
+          name: ubuntu20-build
+      - name: Build dev package
+        run: |
+          zstdcat build.tar.zst | tar x
+          cd build
+          cpack
+      - name: Upload dev package
+        uses: actions/upload-artifact@v3
+        with:
+          name: leap-dev-ubuntu20-amd64
+          path: build/leap-dev*.deb
+
   tests:
     name: Tests
     needs: [d, Build]
@@ -122,9 +140,10 @@ jobs:
         run: |
           # https://github.com/actions/runner/issues/2033  -- need this because of full version label test looking at git revs
           chown -R $(id -u):$(id -g) $PWD
-          zstdcat build.tar.gz | tar x
+          zstdcat build.tar.zst | tar x
           cd build
           ctest --output-on-failure -j $(nproc) -LE "(nonparallelizable_tests|long_running_tests)"
+
   np-tests:
     name: NP Tests
     needs: [d, Build]
@@ -133,30 +152,27 @@ jobs:
       fail-fast: false
       matrix:
         platform: [ubuntu20]
-    runs-on: ubuntu-latest
-    container:
-      image: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
-      options: --init
+    runs-on: ["self-hosted", "enf-x86-midtier"]
     steps:
       - uses: actions/checkout@v3
       - name: Download builddir
         uses: actions/download-artifact@v3
         with:
           name: ${{matrix.platform}}-build
-      - name: Run NP Tests
-        run: |
-          zstdcat build.tar.gz | tar x
-          cd build
-          ctest --output-on-failure -L "nonparallelizable_tests"
-      - name: Bundle logs from failed tests
-        if: failure()
-        run: tar -czf ${{matrix.platform}}-serial-logs.tar.gz build/var build/etc
+      - name: Run tests in parallel containers
+        uses: ./.github/actions/parallel-ctest-containers
+        with:
+          container: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
+          error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd"]'
+          log-tarball-prefix: ${{matrix.platform}}
+          tests: ${{needs.Build.outputs.np-tests}}
       - name: Upload logs from failed tests  
         uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: ${{matrix.platform}}-serial-logs
-          path: ${{matrix.platform}}-serial-logs.tar.gz
+          name: ${{matrix.platform}}-np-logs
+          path: '*-logs.tar.gz'
+
   lr-tests:
     name: LR Tests
     needs: [d, Build]
@@ -165,28 +181,32 @@ jobs:
       fail-fast: false
       matrix:
         platform: [ubuntu20]
-        test-name: ${{fromJSON(needs.Build.outputs.lr-tests)}}
-    runs-on: ubuntu-latest
-    container:
-      image: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
-      options: --init
+    runs-on: ["self-hosted", "enf-x86-lowtier"]
     steps:
       - uses: actions/checkout@v3
       - name: Download builddir
         uses: actions/download-artifact@v3
         with:
           name: ${{matrix.platform}}-build
-      - name: Run ${{matrix.test-name}} Test
-        run: |
-          zstdcat build.tar.gz | tar x
-          cd build
-          ctest --output-on-failure -R ${{matrix.test-name}}
-      - name: Bundle logs from failed tests
-        if: failure()
-        run: tar -czf ${{matrix.platform}}-${{matrix.test-name}}-logs.tar.gz build/var build/etc
-      - name: Upload logs from failed tests  
+      - name: Run tests in parallel containers
+        uses: ./.github/actions/parallel-ctest-containers
+        with:
+          container: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
+          error-log-paths: '["build/etc", "build/var", "build/leap-ignition-wd"]'
+          log-tarball-prefix: ${{matrix.platform}}
+          tests: ${{needs.Build.outputs.lr-tests}}
+      - name: Upload logs from failed tests
         uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: ${{matrix.platform}}-${{matrix.test-name}}-logs
-          path: ${{matrix.platform}}-${{matrix.test-name}}-logs.tar.gz
+          name: ${{matrix.platform}}-lr-logs
+          path: '*-logs.tar.gz'
+
+  all-passing:
+    name: All Required Tests Passed
+    needs: [dev-package, tests, np-tests]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - if: needs.dev-package.result != 'success' || needs.tests.result != 'success' || needs.np-tests.result != 'success'
+        run: false

.gitmodules

@@ -13,9 +13,6 @@
 [submodule "libraries/softfloat"]
 	path = libraries/softfloat
 	url = https://github.com/AntelopeIO/berkeley-softfloat-3
-[submodule "libraries/yubihsm"]
-	path = libraries/yubihsm
-	url = https://github.com/Yubico/yubihsm-shell
 [submodule "tests/abieos"]
 	path = tests/abieos
 	url = https://github.com/AntelopeIO/abieos

CMakeLists.txt

@@ -218,32 +218,16 @@ install(FILES ${CMAKE_BINARY_DIR}/modules/leap-config.cmake DESTINATION ${CMAKE_
 install(FILES ${CMAKE_BINARY_DIR}/modules/EosioTester.cmake DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/cmake/leap COMPONENT dev EXCLUDE_FROM_ALL)
 install(FILES ${CMAKE_SOURCE_DIR}/CMakeModules/EosioCheckVersion.cmake DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/cmake/leap COMPONENT dev EXCLUDE_FROM_ALL)
 
-configure_file(${CMAKE_SOURCE_DIR}/LICENSE
-               ${CMAKE_BINARY_DIR}/licenses/leap/LICENSE COPYONLY)
-configure_file(${CMAKE_SOURCE_DIR}/libraries/softfloat/COPYING.txt
-               ${CMAKE_BINARY_DIR}/licenses/leap/LICENSE.softfloat COPYONLY)
-configure_file(${CMAKE_SOURCE_DIR}/libraries/wasm-jit/LICENSE
-               ${CMAKE_BINARY_DIR}/licenses/leap/LICENSE.wavm COPYONLY)
-configure_file(${CMAKE_SOURCE_DIR}/libraries/fc/secp256k1/secp256k1/COPYING
-               ${CMAKE_BINARY_DIR}/licenses/leap/LICENSE.secp256k1 COPYONLY)
-configure_file(${CMAKE_SOURCE_DIR}/libraries/fc/include/fc/crypto/webauthn_json/license.txt
-               ${CMAKE_BINARY_DIR}/licenses/leap/LICENSE.rapidjson COPYONLY)
-configure_file(${CMAKE_SOURCE_DIR}/libraries/fc/src/network/LICENSE.go
-               ${CMAKE_BINARY_DIR}/licenses/leap/LICENSE.go COPYONLY)
-configure_file(${CMAKE_SOURCE_DIR}/libraries/yubihsm/LICENSE
-               ${CMAKE_BINARY_DIR}/licenses/leap/LICENSE.yubihsm COPYONLY)
-configure_file(${CMAKE_SOURCE_DIR}/libraries/eos-vm/LICENSE
-               ${CMAKE_BINARY_DIR}/licenses/leap/LICENSE.eos-vm COPYONLY)
-
-install(FILES LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/leap/ COMPONENT base)
-install(FILES libraries/softfloat/COPYING.txt DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/leap/ RENAME LICENSE.softfloat COMPONENT base)
-install(FILES libraries/wasm-jit/LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/leap/ RENAME LICENSE.wavm COMPONENT base)
-install(FILES libraries/fc/secp256k1/secp256k1/COPYING DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/leap/ RENAME LICENSE.secp256k1 COMPONENT base)
-install(FILES libraries/fc/include/fc/crypto/webauthn_json/license.txt DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/leap/ RENAME LICENSE.rapidjson COMPONENT base)
-install(FILES libraries/fc/src/network/LICENSE.go DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/leap/ COMPONENT base)
-install(FILES libraries/yubihsm/LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/leap/ RENAME LICENSE.yubihsm COMPONENT base)
-install(FILES libraries/eos-vm/LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/leap/ RENAME LICENSE.eos-vm COMPONENT base)
-install(FILES libraries/fc/libraries/ff/LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/leap/ RENAME LICENSE.libff COMPONENT base)
+configure_file(LICENSE                                                  licenses/leap/LICENSE           COPYONLY)
+configure_file(libraries/softfloat/COPYING.txt                          licenses/leap/LICENSE.softfloat COPYONLY)
+configure_file(libraries/wasm-jit/LICENSE                               licenses/leap/LICENSE.wavm      COPYONLY)
+configure_file(libraries/fc/secp256k1/secp256k1/COPYING                 licenses/leap/LICENSE.secp256k1 COPYONLY)
+configure_file(libraries/fc/include/fc/crypto/webauthn_json/license.txt licenses/leap/LICENSE.rapidjson COPYONLY)
+configure_file(libraries/fc/src/network/LICENSE.go                      licenses/leap/LICENSE.go        COPYONLY)
+configure_file(libraries/eos-vm/LICENSE                                 licenses/leap/LICENSE.eos-vm    COPYONLY)
+configure_file(libraries/fc/libraries/ff/LICENSE                        licenses/leap/LICENSE.libff     COPYONLY)
+
+install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/licenses/leap" DESTINATION "${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/" COMPONENT base)
 
 add_custom_target(dev-install
   COMMAND "${CMAKE_COMMAND}" --build "${CMAKE_BINARY_DIR}"
@@ -252,9 +236,6 @@ add_custom_target(dev-install
   USES_TERMINAL
 )
 
-get_property(_CTEST_CUSTOM_TESTS_IGNORE GLOBAL PROPERTY CTEST_CUSTOM_TESTS_IGNORE)
-file(WRITE "${CMAKE_BINARY_DIR}/CTestCustom.cmake" "SET(CTEST_CUSTOM_TESTS_IGNORE ${_CTEST_CUSTOM_TESTS_IGNORE})")
-
 include(doxygen)
 
 include(package.cmake)