Merge pull request #577 from ucb-bar/small-ci-cleanup

Small CI Improvements
ucb-bar · May 28, 2020 · ea69d6e · ea69d6e
2 parents f711622 + 4972866
commit ea69d6e
Show file tree

Hide file tree

Showing 8 changed files with 19 additions and 11 deletions.
diff --git a/.circleci/build-toolchains.sh b/.circleci/build-toolchains.sh
@@ -14,5 +14,5 @@ if [ ! -d "$HOME/$1-install" ]; then
     cd $HOME
 
     # init all submodules including the tools
-    CHIPYARD_DIR="$LOCAL_CHIPYARD_DIR" NPROC=2 $LOCAL_CHIPYARD_DIR/scripts/build-toolchains.sh $1
+    CHIPYARD_DIR="$LOCAL_CHIPYARD_DIR" NPROC=$CI_MAKE_NPROC $LOCAL_CHIPYARD_DIR/scripts/build-toolchains.sh $1
 fi
diff --git a/.circleci/clean-old-files.sh b/.circleci/clean-old-files.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# clean directories that are older than 30 days
+# clean directories that are older than 14 days
 # argument is used as the directory to look in
 
 age () {
@@ -20,7 +20,7 @@ age () {
 
 for d in $1/*/ ; do
 	DIR_AGE="$(age $d)"
-	if [ $DIR_AGE -ge 30 ]; then
+	if [ $DIR_AGE -ge 14 ]; then
 		echo "Deleting $d since is it $DIR_AGE old"
 		rm -rf $d
 	else

diff --git a/.circleci/defaults.sh b/.circleci/defaults.sh
@@ -18,21 +18,24 @@ clean () {
 }
 
 # make parallelism
-NPROC=8
+CI_MAKE_NPROC=8
+# chosen based on a 24c system shared with 1 other project
+REMOTE_MAKE_NPROC=4
 
 # verilator version
 VERILATOR_VERSION=v4.034
 
 # remote variables
-REMOTE_WORK_DIR=$CI_DIR/$CIRCLE_PROJECT_REPONAME-$CIRCLE_BRANCH-$CIRCLE_SHA1-$CIRCLE_JOB
+REMOTE_PREFIX=$CI_DIR/$CIRCLE_PROJECT_REPONAME-$CIRCLE_BRANCH
+REMOTE_WORK_DIR=$REMOTE_PREFIX-$CIRCLE_SHA1-$CIRCLE_JOB
 REMOTE_RISCV_DIR=$REMOTE_WORK_DIR/riscv-tools-install
 REMOTE_ESP_DIR=$REMOTE_WORK_DIR/esp-tools-install
 REMOTE_CHIPYARD_DIR=$REMOTE_WORK_DIR/chipyard
 REMOTE_SIM_DIR=$REMOTE_CHIPYARD_DIR/sims/verilator
 REMOTE_FIRESIM_DIR=$REMOTE_CHIPYARD_DIR/sims/firesim/sim
 # Disable the supershell to greatly improve the readability of SBT output when captured by Circle CI
 REMOTE_JAVA_ARGS="-Xmx9G -Xss8M -Dsbt.ivy.home=$REMOTE_WORK_DIR/.ivy2 -Dsbt.supershell=false -Dsbt.global.base=$REMOTE_WORK_DIR/.sbt -Dsbt.boot.directory=$REMOTE_WORK_DIR/.sbt/boot"
-REMOTE_VERILATOR_DIR=$CI_DIR/$CIRCLE_PROJECT_REPONAME-$CIRCLE_BRANCH-$CIRCLE_SHA1-verilator-install
+REMOTE_VERILATOR_DIR=$REMOTE_PREFIX-$CIRCLE_SHA1-verilator-install
 
 # local variables (aka within the docker container)
 LOCAL_CHECKOUT_DIR=$HOME/project

diff --git a/.circleci/do-rtl-build.sh b/.circleci/do-rtl-build.sh
@@ -58,7 +58,8 @@ run "export RISCV=\"$TOOLS_DIR\"; \
      export LD_LIBRARY_PATH=\"$LD_LIB_DIR\"; \
      export PATH=\"$REMOTE_VERILATOR_DIR/bin:\$PATH\"; \
      export VERILATOR_ROOT=\"$REMOTE_VERILATOR_DIR\"; \
-     make -j$NPROC -C $REMOTE_SIM_DIR JAVA_ARGS=\"$REMOTE_JAVA_ARGS\" ${mapping[$1]}"
+     export COURSIER_CACHE=\"$REMOTE_WORK_DIR/.coursier-cache\"; \
+     make -j$REMOTE_MAKE_NPROC -C $REMOTE_SIM_DIR JAVA_ARGS=\"$REMOTE_JAVA_ARGS\" ${mapping[$1]}"
 run "rm -rf $REMOTE_CHIPYARD_DIR/project"
 
 # copy back the final build

diff --git a/.circleci/install-verilator.sh b/.circleci/install-verilator.sh
@@ -9,7 +9,9 @@ set -ex
 SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
 source $SCRIPT_DIR/defaults.sh
 
+# clean older directories (delete prior directories related to this branch also)
 run_script $LOCAL_CHIPYARD_DIR/.circleci/clean-old-files.sh $CI_DIR
+run "rm -rf $REMOTE_PREFIX*"
 
 # set stricthostkeychecking to no (must happen before rsync)
 run "echo \"Ping $SERVER\""
@@ -20,4 +22,4 @@ run "git clone http://git.veripool.org/git/verilator $REMOTE_VERILATOR_DIR; \
      autoconf; \
      export VERILATOR_ROOT=$REMOTE_VERILATOR_DIR; \
      ./configure; \
-     make -j$NPROC;"
+     make -j$REMOTE_MAKE_NPROC;"
diff --git a/.circleci/run-firesim-scala-tests.sh b/.circleci/run-firesim-scala-tests.sh
@@ -48,4 +48,5 @@ run "export RISCV=\"$TOOLS_DIR\"; \
      export FIRESIM_ENV_SOURCED=1; \
      export PATH=\"$REMOTE_VERILATOR_DIR/bin:\$PATH\"; \
      export VERILATOR_ROOT=\"$REMOTE_VERILATOR_DIR\"; \
+     export COURSIER_CACHE=\"$REMOTE_WORK_DIR/.coursier-cache\"; \
      make -C $REMOTE_FIRESIM_DIR JAVA_ARGS=\"$REMOTE_JAVA_ARGS\" testOnly ${mapping[$1]}"
diff --git a/.circleci/run-midasexamples-tests.sh b/.circleci/run-midasexamples-tests.sh
@@ -46,4 +46,5 @@ run "export RISCV=\"$TOOLS_DIR\"; \
      export FIRESIM_ENV_SOURCED=1; \
      export PATH=\"$REMOTE_VERILATOR_DIR/bin:\$PATH\"; \
      export VERILATOR_ROOT=\"$REMOTE_VERILATOR_DIR\"; \
+     export COURSIER_CACHE=\"$REMOTE_WORK_DIR/.coursier-cache\"; \
      make -C $REMOTE_FIRESIM_DIR JAVA_ARGS=\"$REMOTE_JAVA_ARGS\" TARGET_PROJECT=midasexamples test"
diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh
@@ -10,11 +10,11 @@ SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
 source $SCRIPT_DIR/defaults.sh
 
 run_bmark () {
-    make run-bmark-tests-fast -j$NPROC -C $LOCAL_SIM_DIR $@
+    make run-bmark-tests-fast -j$CI_MAKE_NPROC -C $LOCAL_SIM_DIR $@
 }
 
 run_asm () {
-    make run-asm-tests-fast -j$NPROC -C $LOCAL_SIM_DIR $@
+    make run-asm-tests-fast -j$CI_MAKE_NPROC -C $LOCAL_SIM_DIR $@
 }
 
 run_both () {
@@ -45,7 +45,7 @@ case $1 in
         export RISCV=$LOCAL_ESP_DIR
         export LD_LIBRARY_PATH=$LOCAL_ESP_DIR/lib
         export PATH=$RISCV/bin:$PATH
-        make run-rv64uv-p-asm-tests -j$NPROC -C $LOCAL_SIM_DIR ${mapping[$1]}
+        make run-rv64uv-p-asm-tests -j$CI_MAKE_NPROC -C $LOCAL_SIM_DIR ${mapping[$1]}
         ;;
     chipyard-gemmini)
         export RISCV=$LOCAL_ESP_DIR