Skip to content

Commit

Permalink
Fix mozilla#3292: Linux debug builds
Browse files Browse the repository at this point in the history
  • Loading branch information
Alexandre Lissy committed Sep 29, 2020
1 parent 0c020d1 commit 56aabaa
Show file tree
Hide file tree
Showing 9 changed files with 154 additions and 14 deletions.
36 changes: 36 additions & 0 deletions taskcluster/.shared.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ kenlm:
tensorflow:
packages_xenial:
apt: 'apt-get -qq update && apt-get -qq -y install realpath build-essential python-virtualenv python-dev python-pip libblas-dev liblapack-dev gfortran wget software-properties-common pixz zip zlib1g-dev unzip'
packages_bionic:
apt: 'apt-get -qq update && apt-get -qq -y install coreutils build-essential python-virtualenv python-dev python-pip libblas-dev liblapack-dev gfortran wget software-properties-common pixz zip zlib1g-dev unzip'
packages_win:
pacman: 'pacman --noconfirm -S patch unzip tar'
msys64: 'ln -s $USERPROFILE/msys64 $TASKCLUSTER_TASK_DIR/msys64'
Expand Down Expand Up @@ -204,6 +206,40 @@ system:
ios_x86_64:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_x86_64/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_x86_64"
tensorflow_dbg:
linux_amd64_cpu:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cpu/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cpu"
linux_amd64_cuda:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cuda/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cuda"
linux_armv7:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.arm/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.arm"
linux_arm64:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.arm64/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.arm64"
darwin_amd64:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.osx/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.osx"
android_arm64:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.android-arm64/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.android-arm64"
android_armv7:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.android-armv7/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.android-armv7"
win_amd64_cpu:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.win/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.win"
win_amd64_cuda:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.win-cuda/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.win-cuda"
ios_arm64:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_arm64/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_arm64"
ios_x86_64:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_x86_64/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_x86_64"
username: 'build-user'
homedir:
linux: '/home/build-user'
Expand Down
17 changes: 17 additions & 0 deletions taskcluster/tf_android-arm64-dbg.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
build:
template_file: generic_tc_caching-linux-opt-base.tyml
cache:
artifact_url: ${system.tensorflow_dbg.android_arm64.url}
artifact_namespace: ${system.tensorflow_dbg.android_arm64.namespace}
system_config:
>
${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
scripts:
setup: "taskcluster/tf_tc-setup.sh --android-arm64"
build: "taskcluster/tf_tc-build.sh --android-arm64 dbg"
package: "taskcluster/tf_tc-package.sh"
maxRunTime: 14400
workerType: "${docker.tfBuild}"
metadata:
name: "TensorFlow Android ARM64 debug"
description: "Building TensorFlow for Android ARM64, debug version"
17 changes: 17 additions & 0 deletions taskcluster/tf_android-armv7-dbg.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
build:
template_file: generic_tc_caching-linux-opt-base.tyml
cache:
artifact_url: ${system.tensorflow_dbg.android_armv7.url}
artifact_namespace: ${system.tensorflow_dbg.android_armv7.namespace}
system_config:
>
${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
scripts:
setup: "taskcluster/tf_tc-setup.sh --android-armv7"
build: "taskcluster/tf_tc-build.sh --android-armv7 dbg"
package: "taskcluster/tf_tc-package.sh"
maxRunTime: 14400
workerType: "${docker.tfBuild}"
metadata:
name: "TensorFlow Android ARMv7 debug"
description: "Building TensorFlow for Android ARMv7, debug version"
17 changes: 17 additions & 0 deletions taskcluster/tf_linux-amd64-cpu-dbg.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
build:
template_file: generic_tc_caching-linux-opt-base.tyml
cache:
artifact_url: ${system.tensorflow_dbg.linux_amd64_cpu.url}
artifact_namespace: ${system.tensorflow_dbg.linux_amd64_cpu.namespace}
system_config:
>
${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
scripts:
setup: "taskcluster/tf_tc-setup.sh"
build: "taskcluster/tf_tc-build.sh --linux-cpu dbg"
package: "taskcluster/tf_tc-package.sh"
maxRunTime: 14400
workerType: "${docker.tfBuild}"
metadata:
name: "TensorFlow Linux AMD64 CPU debug"
description: "Building TensorFlow for Linux/AMD64, CPU only, debug version"
17 changes: 17 additions & 0 deletions taskcluster/tf_linux-amd64-gpu-dbg.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
build:
template_file: generic_tc_caching-linux-opt-base.tyml
cache:
artifact_url: ${system.tensorflow_dbg.linux_amd64_cuda.url}
artifact_namespace: ${system.tensorflow_dbg.linux_amd64_cuda.namespace}
system_config:
>
${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
scripts:
setup: "taskcluster/tf_tc-setup.sh --linux-cuda"
build: "taskcluster/tf_tc-build.sh --linux-cuda dbg"
package: "taskcluster/tf_tc-package.sh"
maxRunTime: 14400
workerType: "${docker.tfBuild}"
metadata:
name: "TensorFlow Linux AMD64 CUDA debug"
description: "Building TensorFlow for Linux/AMD64, CUDA-enabled, debug version"
17 changes: 17 additions & 0 deletions taskcluster/tf_linux-arm64-cpu-dbg.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
build:
template_file: generic_tc_caching-linux-opt-base.tyml
cache:
artifact_url: ${system.tensorflow_dbg.linux_arm64.url}
artifact_namespace: ${system.tensorflow_dbg.linux_arm64.namespace}
system_config:
>
${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
scripts:
setup: "taskcluster/tf_tc-setup.sh"
build: "taskcluster/tf_tc-build.sh --linux-arm64 dbg"
package: "taskcluster/tf_tc-package.sh"
maxRunTime: 14400
workerType: "${docker.tfBuild}"
metadata:
name: "TensorFlow Linux ARM64 Cortex-A53 CPU debug"
description: "Building TensorFlow for Linux ARM64 Cortex-A53, CPU only, debug version"
17 changes: 17 additions & 0 deletions taskcluster/tf_linux-rpi3-cpu-dbg.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
build:
template_file: generic_tc_caching-linux-opt-base.tyml
cache:
artifact_url: ${system.tensorflow_dbg.linux_armv7.url}
artifact_namespace: ${system.tensorflow_dbg.linux_armv7.namespace}
system_config:
>
${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
scripts:
setup: "taskcluster/tf_tc-setup.sh"
build: "taskcluster/tf_tc-build.sh --linux-arm dbg"
package: "taskcluster/tf_tc-package.sh"
maxRunTime: 14400
workerType: "${docker.tfBuild}"
metadata:
name: "TensorFlow Linux RPi3/ARMv7 CPU debug"
description: "Building TensorFlow for Linux RPi3 ARMv7, CPU only, debug version"
22 changes: 13 additions & 9 deletions taskcluster/tf_tc-build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,32 +14,36 @@ pushd ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/
# Force toolchain sync (useful on macOS ?)
bazel ${BAZEL_OUTPUT_USER_ROOT} sync --configure

OPT_OR_DBG=${2:-opt}
MAYBE_DEBUG=$2
OPT_OR_DBG="-c opt"
if [ "${MAYBE_DEBUG}" = "dbg" ]; then
OPT_OR_DBG="-c dbg"
fi;

case "$1" in
"--linux-cpu"|"--darwin-cpu"|"--windows-cpu")
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LIB_CPP_API} ${BUILD_TARGET_LITE_LIB}
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LIB_CPP_API} ${BUILD_TARGET_LITE_LIB}
;;
"--linux-cuda"|"--windows-cuda")
eval "export ${TF_CUDA_FLAGS}" && (echo "" | TF_NEED_CUDA=1 ./configure) && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_CUDA_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BAZEL_OPT_FLAGS} ${BUILD_TARGET_LIB_CPP_API}
eval "export ${TF_CUDA_FLAGS}" && (echo "" | TF_NEED_CUDA=1 ./configure) && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_CUDA_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BAZEL_OPT_FLAGS} ${BUILD_TARGET_LIB_CPP_API}
;;
"--linux-arm")
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
;;
"--linux-arm64")
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
;;
"--android-armv7")
echo "" | TF_SET_ANDROID_WORKSPACE=1 ./configure && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_ANDROID_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
echo "" | TF_SET_ANDROID_WORKSPACE=1 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_ANDROID_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
;;
"--android-arm64")
echo "" | TF_SET_ANDROID_WORKSPACE=1 ./configure && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_ANDROID_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
echo "" | TF_SET_ANDROID_WORKSPACE=1 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_ANDROID_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
;;
"--ios-arm64")
echo "" | TF_NEED_CUDA=0 TF_CONFIGURE_IOS=1 ./configure && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_IOS_ARM64_FLAGS} ${BUILD_TARGET_LITE_LIB}
echo "" | TF_NEED_CUDA=0 TF_CONFIGURE_IOS=1 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_IOS_ARM64_FLAGS} ${BUILD_TARGET_LITE_LIB}
;;
"--ios-x86_64")
echo "" | TF_NEED_CUDA=0 TF_CONFIGURE_IOS=1 ./configure && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_IOS_X86_64_FLAGS} ${BUILD_TARGET_LITE_LIB}
echo "" | TF_NEED_CUDA=0 TF_CONFIGURE_IOS=1 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_IOS_X86_64_FLAGS} ${BUILD_TARGET_LITE_LIB}
;;
esac

Expand Down
8 changes: 3 additions & 5 deletions taskcluster/tf_tc-vars.sh
Original file line number Diff line number Diff line change
Expand Up @@ -137,18 +137,16 @@ fi
# See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html for targetting specific CPUs

if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
CC_OPT_FLAGS="/arch:AVX"
OPT_FLAGS="/arch:AVX"
else
CC_OPT_FLAGS="-mtune=generic -march=x86-64 -msse -msse2 -msse3 -msse4.1 -msse4.2 -mavx"
OPT_FLAGS="-mtune=generic -march=x86-64 -msse -msse2 -msse3 -msse4.1 -msse4.2 -mavx"
fi
BAZEL_OPT_FLAGS=""
for flag in ${CC_OPT_FLAGS};
for flag in ${OPT_FLAGS};
do
BAZEL_OPT_FLAGS="${BAZEL_OPT_FLAGS} --copt=${flag}"
done;

export CC_OPT_FLAGS

BAZEL_OUTPUT_CACHE_DIR="${DS_ROOT_TASK}/.bazel_cache/"
BAZEL_OUTPUT_CACHE_INSTANCE="${BAZEL_OUTPUT_CACHE_DIR}/output/"
mkdir -p ${BAZEL_OUTPUT_CACHE_INSTANCE} || true
Expand Down

0 comments on commit 56aabaa

Please sign in to comment.