[ci] Use sccache and all available CPUs in builds

Jenkinsfile

@@ -45,14 +45,14 @@
 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
 
 // NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. -->
-ci_lint = "tlcpack/ci-lint:v0.68"
-ci_gpu = "tlcpack/ci-gpu:v0.81"
-ci_cpu = "tlcpack/ci-cpu:v0.81"
-ci_wasm = "tlcpack/ci-wasm:v0.71"
-ci_i386 = "tlcpack/ci-i386:v0.74"
-ci_qemu = "tlcpack/ci-qemu:v0.10"
-ci_arm = "tlcpack/ci-arm:v0.07"
-ci_hexagon = "tlcpack/ci-hexagon:v0.01"
+ci_lint = 'tlcpack/ci-lint:v0.68'
+ci_gpu = 'tlcpack/ci-gpu:v0.81'
+ci_cpu = 'tlcpack/ci-cpu:v0.81'
+ci_wasm = 'tlcpack/ci-wasm:v0.71'
+ci_i386 = 'tlcpack/ci-i386:v0.74'
+ci_qemu = 'tlcpack/ci-qemu:v0.10'
+ci_arm = 'tlcpack/ci-arm:v0.07'
+ci_hexagon = 'tlcpack/ci-hexagon:v0.01'
 // <--- End of regex-scanned config.
 
 // Parameters to allow overriding (in Jenkins UI), the images
@@ -194,6 +194,7 @@ stage('Sanity Check') {
   }
 }
 
+
 // Run make. First try to do an incremental make from a previous workspace in hope to
 // accelerate the compilation. If something is wrong, clean the workspace and then
 // build from scratch.
@@ -254,13 +255,13 @@ def python_unittest(image) {
 def fsim_test(image) {
   sh (
     script: "${docker_run} ${image} ./tests/scripts/task_python_vta_fsim.sh",
-    label: 'Run VTA tests in FSIM ',
+    label: 'Run VTA tests in FSIM',
   )
 }
 
 def cmake_build(image, path, make_flag) {
   sh (
-    script: "${docker_run} ${image} ./tests/scripts/task_build.sh ${path} ${make_flag}",
+    script: "${docker_run} ${image} ./tests/scripts/task_build.py --num-executors ${CI_NUM_EXECUTORS} --sccache-bucket tvm-sccache-prod",
     label: 'Run cmake build',
   )
 }

docker/dev_common.sh

@@ -45,7 +45,7 @@ function filter_jenkinsfile() {
 function lookup_image_spec() {
     img_line=$(cat "${GIT_TOPLEVEL}/Jenkinsfile" | filter_jenkinsfile | grep -E "^${1} = ")
     if [ -n "${img_line}" ]; then
-        img_spec=$(echo "${img_line}" | sed -E "s/${1} = \"([^\"]*)\"/\1/")
+        img_spec=$(echo "${img_line}" | sed -E "s/${1} = '([^\"]*)'/\1/")
         has_similar_docker_image=1
         docker inspect "${1}" &>/dev/null || has_similar_docker_image=0
         if [ ${has_similar_docker_image} -ne 0 ]; then

tests/scripts/cmd_utils.py

@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import subprocess
+import os
+import logging
+import sys
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent
+
+
+class RelativePathFilter(logging.Filter):
+    def filter(self, record):
+        path = Path(record.pathname).resolve()
+        record.relativepath = str(path.relative_to(REPO_ROOT))
+        return True
+
+
+def init_log():
+    logging.basicConfig(
+        format="[%(relativepath)s:%(lineno)d %(levelname)-1s] %(message)s", level=logging.INFO
+    )
+
+    # Flush on every log call (logging and then calling subprocess.run can make
+    # the output look confusing)
+    logging.root.handlers[0].addFilter(RelativePathFilter())
+    logging.root.handlers[0].flush = sys.stderr.flush
+
+
+class Sh:
+    def __init__(self, env=None):
+        self.env = os.environ.copy()
+        if env is not None:
+            self.env.update(env)
+
+    def run(self, cmd: str, **kwargs):
+        logging.info(f"+ {cmd}")
+        if "check" not in kwargs:
+            kwargs["check"] = True
+        if "shell" not in kwargs:
+            kwargs["shell"] = True
+        if "env" not in kwargs:
+            kwargs["env"] = self.env
+
+        subprocess.run(cmd, **kwargs)

tests/scripts/task_build.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import argparse
+import shutil
+import os
+import logging
+import multiprocessing
+
+from pathlib import Path
+from cmd_utils import Sh, init_log, REPO_ROOT
+
+
+if __name__ == "__main__":
+    init_log()
+
+    parser = argparse.ArgumentParser(description="List pytest nodeids for a folder")
+    parser.add_argument("--sccache-bucket", required=False, help="sccache bucket name")
+    parser.add_argument("--num-executors", required=True, help="number of Jenkins executors")
+    parser.add_argument("--build-dir", default="build", help="build folder")
+    args = parser.parse_args()
+
+    env = {"VTA_HW_PATH": str(Path(os.getcwd()) / "3rdparty" / "vta-hw")}
+    sccache_exe = shutil.which("sccache")
+
+    use_sccache = sccache_exe is not None and args.sccache_bucket is not None
+    build_dir = Path(os.getcwd()) / args.build_dir
+    build_dir = build_dir.relative_to(REPO_ROOT)
+
+    if use_sccache:
+        env["SCCACHE_BUCKET"] = args.sccache_bucket
+        env["CXX"] = "/opt/sccache/c++"
+        env["CC"] = "/opt/sccache/cc"
+
+        logging.info(f"Using sccache bucket: {args.sccache_bucket}")
+    else:
+        if sccache_exe is None:
+            reason = "'sccache' executable not found"
+        elif args.sccache_bucket is None:
+            reason = "'sccache' executable not found"
+        else:
+            reason = "<unknown>"
+        logging.info(f"Not using sccache, reason: {reason}")
+
+    sh = Sh(env)
+
+    if use_sccache:
+        sh.run("sccache --start-server", check=False)
+        logging.info("===== sccache stats =====")
+        sh.run("sccache --show-stats")
+
+    executors = int(args.num_executors)
+    nproc = multiprocessing.cpu_count()
+
+    available_cpus = nproc // executors
+    num_cpus = max(available_cpus, 1)
+
+    sh.run("cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo ..", cwd=build_dir)
+    sh.run(f"cmake --build . -- VERBOSE=1 -j{num_cpus}", cwd=build_dir)
+
+    if use_sccache:
+        logging.info("===== sccache stats =====")
+        sh.run("sccache --show-stats")

tests/scripts/task_config_build_arm.sh

@@ -29,7 +29,6 @@ echo set\(USE_MICRO ON\) >> config.cmake
 echo set\(USE_MICRO_STANDALONE_RUNTIME ON\) >> config.cmake
 echo set\(USE_PROFILER ON\) >> config.cmake
 echo set\(USE_LLVM llvm-config-8\) >> config.cmake
-echo set\(CMAKE_CXX_COMPILER g++\) >> config.cmake
 echo set\(CMAKE_CXX_FLAGS -Werror\) >> config.cmake
 echo set\(USE_VTA_FSIM ON\) >> config.cmake
 echo set\(USE_ARM_COMPUTE_LIB ON\) >> config.cmake

tests/scripts/task_config_build_cpu.sh

@@ -33,7 +33,6 @@ echo set\(USE_LLVM llvm-config-11\) >> config.cmake
 echo set\(USE_NNPACK ON\) >> config.cmake
 echo set\(NNPACK_PATH /NNPACK/build/\) >> config.cmake
 echo set\(USE_ANTLR ON\) >> config.cmake
-echo set\(CMAKE_CXX_COMPILER g++\) >> config.cmake
 echo set\(CMAKE_CXX_FLAGS -Werror\) >> config.cmake
 echo set\(HIDE_PRIVATE_SYMBOLS ON\) >> config.cmake
 echo set\(USE_VTA_TSIM ON\) >> config.cmake

tests/scripts/task_config_build_gpu.sh

@@ -41,7 +41,6 @@ echo set\(USE_PROFILER ON\) >> config.cmake
 echo set\(USE_ANTLR ON\) >> config.cmake
 echo set\(USE_VTA_FSIM ON\) >> config.cmake
 echo set\(USE_BLAS openblas\) >> config.cmake
-echo set\(CMAKE_CXX_COMPILER g++\) >> config.cmake
 echo set\(CMAKE_CXX_FLAGS -Werror\) >> config.cmake
 echo set\(USE_TENSORRT_CODEGEN ON\) >> config.cmake
 echo set\(USE_LIBBACKTRACE AUTO\) >> config.cmake

tests/scripts/task_config_build_i386.sh

@@ -29,7 +29,6 @@ echo set\(USE_MICRO ON\) >> config.cmake
 echo set\(USE_MICRO_STANDALONE_RUNTIME ON\) >> config.cmake
 echo set\(USE_PROFILER ON\) >> config.cmake
 echo set\(USE_LLVM llvm-config-4.0\) >> config.cmake
-echo set\(CMAKE_CXX_COMPILER g++\) >> config.cmake
 echo set\(CMAKE_CXX_FLAGS -Werror\) >> config.cmake
 echo set\(USE_VTA_FSIM ON\) >> config.cmake
 echo set\(USE_VTA_TSIM ON\) >> config.cmake

tests/scripts/task_config_build_qemu.sh

@@ -28,7 +28,6 @@ echo set\(USE_MICRO ON\) >> config.cmake
 echo set\(USE_CMSISNN ON\) >> config.cmake
 echo set\(USE_ETHOSU ON\) >> config.cmake
 echo set\(USE_LLVM llvm-config-10\) >> config.cmake
-echo set\(CMAKE_CXX_COMPILER g++\) >> config.cmake
 echo set\(CMAKE_CXX_FLAGS -Werror\) >> config.cmake
 echo set\(HIDE_PRIVATE_SYMBOLS ON\) >> config.cmake
 echo set\(USE_CCACHE OFF\) >> config.cmake

tests/scripts/task_config_build_wasm.sh

@@ -29,7 +29,6 @@ echo set\(USE_MICRO_STANDALONE_RUNTIME ON\) >> config.cmake
 echo set\(USE_PROFILER ON\) >> config.cmake
 echo set\(USE_LLVM llvm-config-11\) >> config.cmake
 echo set\(USE_ANTLR ON\) >> config.cmake
-echo set\(CMAKE_CXX_COMPILER g++\) >> config.cmake
 echo set\(CMAKE_CXX_FLAGS -Werror\) >> config.cmake
 echo set\(HIDE_PRIVATE_SYMBOLS ON\) >> config.cmake
 echo set\(USE_VTA_FSIM ON\) >> config.cmake

tests/scripts/task_show_node_info.sh

@@ -19,11 +19,11 @@
 set -x
 
 echo "===== JENKINS INFO ====="
-echo "$NODE_NAME"
-echo "$EXECUTOR_NUMBER"
-echo "$WORKSPACE"
-echo "$BUILD_NUMBER"
-echo "$WORKSPACE"
+echo "NODE_NAME=$NODE_NAME"
+echo "EXECUTOR_NUMBER=$EXECUTOR_NUMBER"
+echo "WORKSPACE=$WORKSPACE"
+echo "BUILD_NUMBER=$BUILD_NUMBER"
+echo "WORKSPACE=$WORKSPACE"
 
 echo "===== EC2 INFO ====="
 function ec2_metadata() {