-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Optimised Windows CI/CD setup by internalising and caching chocolatey packages Fixes #397
- Loading branch information
1 parent
f9ac03c
commit 1290813
Showing
7 changed files
with
294 additions
and
254 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,180 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| shopt -s globstar | ||
| shopt -s nullglob | ||
|
|
||
| # Quote the heredoc to prevent shell expansion | ||
| cat << "EOF" | ||
| default: | ||
| interruptible: true | ||
| variables: | ||
| GH_PROJECT_PATH: "MatrixAI/${CI_PROJECT_NAME}" | ||
| GH_PROJECT_URL: "https://${GITHUB_TOKEN}@github.com/${GH_PROJECT_PATH}.git" | ||
| GIT_SUBMODULE_STRATEGY: "recursive" | ||
| # Cache .npm | ||
| NPM_CONFIG_CACHE: "./tmp/npm" | ||
| # Prefer offline node module installation | ||
| NPM_CONFIG_PREFER_OFFLINE: "true" | ||
| # `ts-node` has its own cache | ||
| # It must use an absolute path, otherwise ts-node calls will CWD | ||
| TS_CACHED_TRANSPILE_CACHE: "${CI_PROJECT_DIR}/tmp/ts-node-cache" | ||
| TS_CACHED_TRANSPILE_PORTABLE: "true" | ||
| # Homebrew cache only used by macos runner | ||
| HOMEBREW_CACHE: "${CI_PROJECT_DIR}/tmp/Homebrew" | ||
| # Cached directories shared between jobs & pipelines per-branch per-runner | ||
| cache: | ||
| key: $CI_COMMIT_REF_SLUG | ||
| when: 'always' | ||
| paths: | ||
| - ./tmp/npm/ | ||
| - ./tmp/ts-node-cache/ | ||
| # Homebrew cache is only used by the macos runner | ||
| - ./tmp/Homebrew | ||
| # Chocolatey cache is only used by the windows runner | ||
| - ./tmp/chocolatey/ | ||
| # `jest` cache is configured in jest.config.js | ||
| - ./tmp/jest/ | ||
| stages: | ||
| - build # Cross-platform library compilation, unit tests | ||
| image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner | ||
| EOF | ||
|
|
||
| printf "\n" | ||
|
|
||
| # Each test directory has its own job | ||
| for test_dir in tests/**/*/; do | ||
| test_files=("$test_dir"*.test.ts) | ||
| if [ ${#test_files[@]} -eq 0 ]; then | ||
| continue | ||
| fi | ||
| # Remove trailing slash | ||
| test_dir="${test_dir%\/}" | ||
| # Remove `tests/` prefix | ||
| test_dir="${test_dir#*/}" | ||
| cat << EOF | ||
| build:linux $test_dir: | ||
| stage: build | ||
| needs: [] | ||
| script: | ||
| - > | ||
| nix-shell --run ' | ||
| npm test -- --ci --coverage --runInBand ${test_files[@]}; | ||
| ' | ||
| artifacts: | ||
| when: always | ||
| reports: | ||
| junit: | ||
| - ./tmp/junit/junit.xml | ||
| coverage_report: | ||
| coverage_format: cobertura | ||
| path: ./tmp/coverage/cobertura-coverage.xml | ||
| coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/' | ||
| EOF | ||
| printf "\n" | ||
| done | ||
|
|
||
| # All top-level test files are accumulated into 1 job | ||
| test_files=(tests/*.test.ts) | ||
| cat << EOF | ||
| build:linux index: | ||
| stage: build | ||
| needs: [] | ||
| script: | ||
| - > | ||
| nix-shell --run ' | ||
| npm test -- --ci --coverage --runInBand ${test_files[@]}; | ||
| ' | ||
| artifacts: | ||
| when: always | ||
| reports: | ||
| junit: | ||
| - ./tmp/junit/junit.xml | ||
| coverage_report: | ||
| coverage_format: cobertura | ||
| path: ./tmp/coverage/cobertura-coverage.xml | ||
| coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/' | ||
| EOF | ||
|
|
||
| printf "\n" | ||
|
|
||
| # Using shards to optimise tests | ||
| # In the future we can incorporate test durations rather than using | ||
| # a static value for the parallel keyword | ||
|
|
||
| # Number of parallel shards to split the test suite into | ||
| CI_PARALLEL=2 | ||
|
|
||
| cat << "EOF" | ||
| build:windows: | ||
| stage: build | ||
| needs: [] | ||
| EOF | ||
| cat << EOF | ||
| parallel: $CI_PARALLEL | ||
| EOF | ||
| cat << "EOF" | ||
| tags: | ||
| - windows | ||
| before_script: | ||
| - .\scripts\choco-install.ps1 | ||
| - refreshenv | ||
| script: | ||
| - npm config set msvs_version 2019 | ||
| - npm install --ignore-scripts | ||
| - $env:Path = "$(npm bin);" + $env:Path | ||
| - npm test -- --ci --coverage --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL --maxWorkers=50% | ||
| artifacts: | ||
| when: always | ||
| reports: | ||
| junit: | ||
| - ./tmp/junit/junit.xml | ||
| coverage_report: | ||
| coverage_format: cobertura | ||
| path: ./tmp/coverage/cobertura-coverage.xml | ||
| coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/' | ||
| EOF | ||
|
|
||
| printf "\n" | ||
|
|
||
| cat << "EOF" | ||
| build:macos: | ||
| stage: build | ||
| needs: [] | ||
| EOF | ||
| cat << EOF | ||
| parallel: $CI_PARALLEL | ||
| EOF | ||
| cat << "EOF" | ||
| tags: | ||
| - shared-macos-amd64 | ||
| image: macos-11-xcode-12 | ||
| variables: | ||
| HOMEBREW_NO_INSTALL_UPGRADE: "true" | ||
| HOMEBREW_NO_INSTALL_CLEANUP: "true" | ||
| HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: "true" | ||
| HOMEBREW_NO_AUTO_UPDATE: "true" | ||
| before_script: | ||
| - eval "$(brew shellenv)" | ||
| - brew install node@16 | ||
| - brew link --overwrite node@16 | ||
| - hash -r | ||
| script: | ||
| - npm install --ignore-scripts | ||
| - export PATH="$(npm bin):$PATH" | ||
| - npm test -- --ci --coverage --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL --maxWorkers=50% | ||
| artifacts: | ||
| when: always | ||
| reports: | ||
| junit: | ||
| - ./tmp/junit/junit.xml | ||
| coverage_report: | ||
| coverage_format: cobertura | ||
| path: ./tmp/coverage/cobertura-coverage.xml | ||
| coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/' | ||
| EOF | ||
|
|
||
| printf "\n" |
Oops, something went wrong.