From d845fa152f46f805b6cc1184d92221ac792cf03b Mon Sep 17 00:00:00 2001 From: Matt McCormick Date: Fri, 23 Aug 2024 13:31:09 -0400 Subject: [PATCH 1/5] chore(native): bump ITK, DCMTK to match Docker --- itk_wasm_env.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/itk_wasm_env.bash b/itk_wasm_env.bash index 1a19b9ad1..84846a87c 100755 --- a/itk_wasm_env.bash +++ b/itk_wasm_env.bash @@ -11,10 +11,10 @@ if [$OSTYPE == "cygwin"] || [$OSTYPE == "msys"] || [$OSTYPE =="win32"]; then fi export ITK_WASM_DCMTK_REPOSITORY=${ITK_WASM_DCMTK_REPOSITORY:-"https://github.com/InsightSoftwareConsortium/DCMTK"} -export ITK_WASM_DCMTK_GIT_TAG=${ITK_WASM_DCMTK_GIT_TAG:-"fe7cff5de40b67ae0490d476ddf17689c06bcaf4"} +export ITK_WASM_DCMTK_GIT_TAG=${ITK_WASM_DCMTK_GIT_TAG:-"93e6d1b20b476c388f0379245c8c6e96af7f16f4"} export ITK_WASM_ITK_REPOSITORY=${ITK_WASM_ITK_REPOSITORY:-"https://github.com/KitwareMedical/ITK"} -export ITK_WASM_ITK_BRANCH=${ITK_WASM_ITK_BRANCH:-"itkwasm-2024-05-20-5db055d7ad3b-1"} +export ITK_WASM_ITK_BRANCH=${ITK_WASM_ITK_BRANCH:-"itkwasm-2024-05-20-5db055d7ad3b-3"} export ITK_WASM_NATIVE_WORKSPACE=${ITK_WASM_NATIVE_WORKSPACE:-$(pwd)/native} From 859414a1f5a092cdbc89bb7ae6c01eefcb7e39e4 Mon Sep 17 00:00:00 2001 From: Matt McCormick Date: Fri, 23 Aug 2024 14:03:12 -0400 Subject: [PATCH 2/5] build(native): add patch to fix ITKTotalVariation build on Linux Adds: https://github.com/InsightSoftwareConsortium/ITKTotalVariation/pull/52 --- itk_wasm_env.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/itk_wasm_env.bash b/itk_wasm_env.bash index 84846a87c..11a717182 100755 --- a/itk_wasm_env.bash +++ b/itk_wasm_env.bash @@ -14,7 +14,7 @@ export ITK_WASM_DCMTK_REPOSITORY=${ITK_WASM_DCMTK_REPOSITORY:-"https://github.co export ITK_WASM_DCMTK_GIT_TAG=${ITK_WASM_DCMTK_GIT_TAG:-"93e6d1b20b476c388f0379245c8c6e96af7f16f4"} export ITK_WASM_ITK_REPOSITORY=${ITK_WASM_ITK_REPOSITORY:-"https://github.com/KitwareMedical/ITK"} -export ITK_WASM_ITK_BRANCH=${ITK_WASM_ITK_BRANCH:-"itkwasm-2024-05-20-5db055d7ad3b-3"} +export ITK_WASM_ITK_BRANCH=${ITK_WASM_ITK_BRANCH:-"itkwasm-2024-05-20-5db055d7ad3b-4"} export ITK_WASM_NATIVE_WORKSPACE=${ITK_WASM_NATIVE_WORKSPACE:-$(pwd)/native} From 90b6e672f96b30da66bebbebb05696dfeefc7fa8 Mon Sep 17 00:00:00 2001 From: Matt McCormick Date: Fri, 23 Aug 2024 16:18:38 -0400 Subject: [PATCH 3/5] build: re-use ITK, DCMTK source locations in native, docker builds Also add the ability to drive via pixi. --- .gitignore | 2 +- docs/development/hacking_itk_wasm.md | 4 ++-- itk_wasm_env.bash | 4 ++-- pixi.toml | 16 ++++++++++++++++ src/docker/itk-wasm-base/Dockerfile | 11 +++++------ 5 files changed, 26 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 6c68c8bb7..e2952ef66 100644 --- a/.gitignore +++ b/.gitignore @@ -102,4 +102,4 @@ packages/transform-io/typescript/demo-app/ *.egg-info native/ - +src/docker/itk-wasm-base/itk_wasm_env_vars.sh diff --git a/docs/development/hacking_itk_wasm.md b/docs/development/hacking_itk_wasm.md index 33a0bf1a5..491f7a9bc 100644 --- a/docs/development/hacking_itk_wasm.md +++ b/docs/development/hacking_itk_wasm.md @@ -69,7 +69,7 @@ To pull the `latest` the build environment Docker images, To build the `latest` build environment Docker images from the Docker configuration and local C++ core, ```sh -./src/docker/build.sh --with-debug +pixi run build-docker-images --with-debug ``` The `--with-debug` flag will also build the `latest-debug` tagged images. @@ -242,4 +242,4 @@ and rebuild whenever the documentation changes. [standard GitHub contribution best practices]: https://docs.itk.org/en/latest/contributing/index.html [web3.storage]: https://web3.storage/ [Windows Git Bash]: https://gitforwindows.org/ -[WSL]: https://learn.microsoft.com/en-us/windows/wsl/install \ No newline at end of file +[WSL]: https://learn.microsoft.com/en-us/windows/wsl/install diff --git a/itk_wasm_env.bash b/itk_wasm_env.bash index 11a717182..aed6460aa 100755 --- a/itk_wasm_env.bash +++ b/itk_wasm_env.bash @@ -5,7 +5,7 @@ function die() { exit 1 } -if [$OSTYPE == "cygwin"] || [$OSTYPE == "msys"] || [$OSTYPE =="win32"]; then +if test "$OSTYPE" = "cygwin" || test "$OSTYPE" = "msys" || test "$OSTYPE" = "win32"; then echo "Windows platform detected ... adding \"/Zc:__cplusplus /DNOMINMAX\" to \$CXXFLAGS" export CXXFLAGS="/Zc:__cplusplus /DNOMINMAX" fi @@ -27,4 +27,4 @@ export ITK_WASM_DICOM_TEST_DATA_HASH=${ITK_WASM_DICOM_TEST_DATA_HASH:-$(cat pack export ITK_WASM_DICOM_TEST_DATA_URLS=${ITK_WASM_DICOM_TEST_DATA_URLS:-$(cat packages/dicom/package.json | jq -e -r '."itk-wasm"."test-data-urls" | join(" ")')} export ITK_WASM_MESH_IO_TEST_DATA_HASH=${ITK_WASM_MESH_IO_TEST_DATA_HASH:-$(cat packages/mesh-io/package.json | jq -e -r '."itk-wasm"."test-data-hash"')} -export ITK_WASM_MESH_IO_TEST_DATA_URLS=${ITK_WASM_MESH_IO_TEST_DATA_URLS:-$(cat packages/mesh-io/package.json | jq -e -r '."itk-wasm"."test-data-urls" | join(" ")')} \ No newline at end of file +export ITK_WASM_MESH_IO_TEST_DATA_URLS=${ITK_WASM_MESH_IO_TEST_DATA_URLS:-$(cat packages/mesh-io/package.json | jq -e -r '."itk-wasm"."test-data-urls" | join(" ")')} diff --git a/pixi.toml b/pixi.toml index 8a71e4449..081c5ad42 100644 --- a/pixi.toml +++ b/pixi.toml @@ -29,6 +29,22 @@ depends-on = ["pnpm-install"] outputs = ["packages/mesh-io/test/data.tar.gz"] description = "Download mesh-io test data" +[tasks.export-itk-wasm-env-vars] +cmd = "bash -c ./itk_wasm_env.bash && env | grep ITK_WASM | grep -v TEST > ./src/docker/itk-wasm-base/itk_wasm_env_vars.sh" +outputs = ["src/docker/itk-wasm-base/itk_wasm_env_vars.*"] +inputs = ["itk_wasm_env.*"] +description = "Provide ITK_WASM environmental variables to the Docker image" + +[tasks.update-default-image-tag] +cmd = "sed -i \"s/const defaultImageTag = '.*'/const defaultImageTag = '$new_tag'/g\" packages/core/typescript/itk-wasm/src/cli/default-image-tag.js" +outputs = ["packages/core/typescript/itk-wasm/src/cli/default-image-tag.*"] +description = "Update the default Docker image tag" + +[tasks.build-docker-images] +cmd = "src/docker/build.sh" +description = "Build the ITK-Wasm docker images" +depends-on = ["clean", "export-itk-wasm-env-vars", "update-default-image-tag"] + [dependencies] pnpm = ">=9.7.1,<10" nodejs = ">=22.6.0,<23" diff --git a/src/docker/itk-wasm-base/Dockerfile b/src/docker/itk-wasm-base/Dockerfile index 67c47655c..1f0534d37 100644 --- a/src/docker/itk-wasm-base/Dockerfile +++ b/src/docker/itk-wasm-base/Dockerfile @@ -7,6 +7,8 @@ LABEL org.opencontainers.image.source="https://github.com/InsightSoftwareConsort WORKDIR / +COPY itk_wasm_env_vars.sh / + # Note: on entry, emsdk will prepend to the path with its own node, so add to an earlier path ENV NODE_TAG v20.12.2 RUN curl -LO https://nodejs.org/dist/${NODE_TAG}/node-${NODE_TAG}-linux-x64.tar.xz && \ @@ -33,19 +35,16 @@ RUN curl -L https://api.github.com/repos/facebook/zstd/tarball/${zstd_GIT_TAG} | cd .. && \ rm -rf zstd-build zstd -# tag commit date: 2024-08-16 -ENV ITK_GIT_TAG eddb514e52a23e2b0987b71fda59780c6ad209ab -ENV ITK_GIT_BRANCH itkwasm-2024-05-20-5db055d7ad3b-3 -RUN git clone --branch $ITK_GIT_BRANCH --single-branch --depth 1 https://github.com/KitwareMedical/ITK.git && \ +RUN . /itk_wasm_env_vars.sh && git clone --branch $ITK_WASM_ITK_BRANCH --single-branch --depth 1 $ITK_WASM_ITK_REPOSITORY && \ sed -i -e '/^option(OPJ_USE_THREAD/c\option(OPJ_USE_THREAD "use threads" OFF)' \ /ITK/Modules/ThirdParty/GDCM/src/gdcm/Utilities/gdcmopenjpeg/src/lib/openjp2/CMakeLists.txt # Modify CMake variable to use patched DCMTK library # GIT_TAG refers to DCMTK branch: 20240311_DCMTK_PATCHES_FOR_ITK-Wasm-1 -RUN sed -i -e '/^set(DCMTK_GIT_REPOSITORY/c\set(DCMTK_GIT_REPOSITORY "https://github.com/InsightSoftwareConsortium/DCMTK.git")' \ +RUN . /itk_wasm_env_vars.sh && sed -i -e "/^set(DCMTK_GIT_REPOSITORY/c\set(DCMTK_GIT_REPOSITORY \"${ITK_WASM_DCMTK_REPOSITORY}\")" \ /ITK/Modules/ThirdParty/DCMTK/DCMTKGitTag.cmake -RUN sed -i -e '/^set(DCMTK_GIT_TAG/c\set(DCMTK_GIT_TAG "93e6d1b20b476c388f0379245c8c6e96af7f16f4")' \ +RUN . /itk_wasm_env_vars.sh && sed -i -e "/^set(DCMTK_GIT_TAG/c\set(DCMTK_GIT_TAG \"${ITK_WASM_DCMTK_GIT_TAG}\")" \ /ITK/Modules/ThirdParty/DCMTK/DCMTKGitTag.cmake RUN if test -e $WASI_SDK_PATH/share/cmake/wasi-sdk.cmake; then sed -i '/cmake_minimum_required/d' $WASI_SDK_PATH/share/cmake/wasi-sdk.cmake; fi From 6d25c4a949c0410db685b7f41f25cbc5778e2612 Mon Sep 17 00:00:00 2001 From: Matt McCormick Date: Sat, 24 Aug 2024 13:26:25 -0400 Subject: [PATCH 4/5] ci(toolchains): pixi run export-itk-wasm-env-vars --- .github/workflows/toolchains.yml | 126 ++++++++++++++++--------------- 1 file changed, 66 insertions(+), 60 deletions(-) diff --git a/.github/workflows/toolchains.yml b/.github/workflows/toolchains.yml index 0d37dbe8b..55ee975e7 100644 --- a/.github/workflows/toolchains.yml +++ b/.github/workflows/toolchains.yml @@ -17,41 +17,44 @@ jobs: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 - - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - - - name: Pull latest Docker images - run: | - ./src/docker/pull.sh --no-debug - - - uses: actions/download-artifact@v4 - continue-on-error: true - with: - name: docker-cache - path: ./docker-cache - - - name: Build itk-wasm-base - run: | - if [[ -e ./docker-cache/itk-wasm-emscripten-base.tar.xz ]]; then xz -d -k < ./docker-cache/itk-wasm-emscripten-base.tar.xz | docker load ; fi - ./src/docker/itk-wasm-base/build.sh - mkdir -p ./docker-cache - docker export itkwasm/emscripten-base:latest | xz -e9 -T0 > ./docker-cache/itk-wasm-emscripten-base.tar.xz - - - name: Build itk-wasm-base - run: | - if [[ -e ./docker-cache/itk-wasm-wasi-base.tar.xz ]]; then xz -d -k < ./docker-cache/itk-wasm-wasi-base.tar.xz | docker load ; fi - ./src/docker/itk-wasm-base/build.sh --with-wasi - mkdir -p ./docker-cache - docker export itkwasm/wasi-base:latest | xz -e9 -T0 > ./docker-cache/itk-wasm-wasi-base.tar.xz - - - name: Cache docker results - uses: actions/upload-artifact@v4 - with: - name: docker-cache - path: ./docker-cache - retention-days: 7 + - uses: actions/checkout@v4 + + - uses: prefix-dev/setup-pixi@v0.8.1 + - run: pixi run export-itk-wasm-env-vars + + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + + - name: Pull latest Docker images + run: | + ./src/docker/pull.sh --no-debug + + - uses: actions/download-artifact@v4 + continue-on-error: true + with: + name: docker-cache + path: ./docker-cache + + - name: Build itk-wasm-base + run: | + if [[ -e ./docker-cache/itk-wasm-emscripten-base.tar.xz ]]; then xz -d -k < ./docker-cache/itk-wasm-emscripten-base.tar.xz | docker load ; fi + ./src/docker/itk-wasm-base/build.sh + mkdir -p ./docker-cache + docker export itkwasm/emscripten-base:latest | xz -e9 -T0 > ./docker-cache/itk-wasm-emscripten-base.tar.xz + + - name: Build itk-wasm-base + run: | + if [[ -e ./docker-cache/itk-wasm-wasi-base.tar.xz ]]; then xz -d -k < ./docker-cache/itk-wasm-wasi-base.tar.xz | docker load ; fi + ./src/docker/itk-wasm-base/build.sh --with-wasi + mkdir -p ./docker-cache + docker export itkwasm/wasi-base:latest | xz -e9 -T0 > ./docker-cache/itk-wasm-wasi-base.tar.xz + + - name: Cache docker results + uses: actions/upload-artifact@v4 + with: + name: docker-cache + path: ./docker-cache + retention-days: 7 build-itk-wasm: name: "Build itk-wasm" @@ -59,28 +62,31 @@ jobs: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 - - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - - - name: Pull latest Docker images - run: | - ./src/docker/pull.sh --no-debug - - - uses: actions/download-artifact@v4 - continue-on-error: true - with: - name: docker-cache - path: ./docker-cache - - - name: Build itk-wasm - run: | - sudo mkdir -p /var/lib/docker/tmp - if [[ -e ./docker-cache/itk-wasm-emscripten-base.tar.xz ]]; then xz -d -k < ./docker-cache/itk-wasm-emscripten-base.tar.xz | docker import - itkwasm/emscripten:latest ; fi - ./src/docker/itk-wasm/build.sh - - - name: Build itk-wasi - run: | - if [[ -e ./docker-cache/itk-wasm-wasi-base.tar.xz ]]; then xz -d -k < ./docker-cache/itk-wasm-wasi-base.tar.xz | docker import - itkwasm/wasi:latest ; fi - ./src/docker/itk-wasm/build.sh --with-wasi + - uses: actions/checkout@v4 + + - uses: prefix-dev/setup-pixi@v0.8.1 + - run: pixi run export-itk-wasm-env-vars + + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + + - name: Pull latest Docker images + run: | + ./src/docker/pull.sh --no-debug + + - uses: actions/download-artifact@v4 + continue-on-error: true + with: + name: docker-cache + path: ./docker-cache + + - name: Build itk-wasm + run: | + sudo mkdir -p /var/lib/docker/tmp + if [[ -e ./docker-cache/itk-wasm-emscripten-base.tar.xz ]]; then xz -d -k < ./docker-cache/itk-wasm-emscripten-base.tar.xz | docker import - itkwasm/emscripten:latest ; fi + ./src/docker/itk-wasm/build.sh + + - name: Build itk-wasi + run: | + if [[ -e ./docker-cache/itk-wasm-wasi-base.tar.xz ]]; then xz -d -k < ./docker-cache/itk-wasm-wasi-base.tar.xz | docker import - itkwasm/wasi:latest ; fi + ./src/docker/itk-wasm/build.sh --with-wasi From f81d1deb6f99de9bb09d422655bb7b1618ceedf0 Mon Sep 17 00:00:00 2001 From: Matt McCormick Date: Sat, 24 Aug 2024 16:12:15 -0400 Subject: [PATCH 5/5] build(native): update MSVC flags Co-authored-by: Shreeraj Jadhav --- itk_wasm_env.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/itk_wasm_env.bash b/itk_wasm_env.bash index aed6460aa..dad79139a 100755 --- a/itk_wasm_env.bash +++ b/itk_wasm_env.bash @@ -7,7 +7,7 @@ function die() { if test "$OSTYPE" = "cygwin" || test "$OSTYPE" = "msys" || test "$OSTYPE" = "win32"; then echo "Windows platform detected ... adding \"/Zc:__cplusplus /DNOMINMAX\" to \$CXXFLAGS" - export CXXFLAGS="/Zc:__cplusplus /DNOMINMAX" + export CXXFLAGS="/Zc:__cplusplus /Zc:preprocessor /DNOMINMAX" fi export ITK_WASM_DCMTK_REPOSITORY=${ITK_WASM_DCMTK_REPOSITORY:-"https://github.com/InsightSoftwareConsortium/DCMTK"}