diff --git a/hack/jenkins/build.sh b/hack/jenkins/build.sh deleted file mode 100755 index ddd4d14c674c6..0000000000000 --- a/hack/jenkins/build.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# kubernetes-build job: Triggered by github checkins on a 5 minute -# poll. We abort this job if it takes longer than 10m. (Typically this -# job takes about ~5m as of 0.8.0, but it's actually not completely -# hermetic right now due to things like the golang image. It can take -# ~8m if you force it to be totally hermetic.) - -set -o errexit -set -o nounset -set -o pipefail -set -o xtrace - -# !!! ALERT !!! Jenkins default $HOME is /var/lib/jenkins, which is -# global across jobs. We change $HOME instead to ${WORKSPACE}, which -# is an incoming variable Jenkins provides us for this job's scratch -# space. -export HOME=${WORKSPACE} # Nothing should want Jenkins $HOME -export PATH=${PATH}:/usr/local/go/bin - -# Skip gcloud update checking -export CLOUDSDK_COMPONENT_MANAGER_DISABLE_UPDATE_CHECK=true - -: "${KUBE_RELEASE_RUN_TESTS:="n"}" -export KUBE_RELEASE_RUN_TESTS - -# Clean stuff out. Assume the last build left the tree in an odd -# state. -rm -rf ~/.kube* -make clean - -# Uncomment if you want to purge the Docker cache completely each -# build. It costs about 150s each build to pull the golang image and -# rebuild the kube-build:cross image, but these rarely change. -# docker ps -aq | xargs -r docker rm -# docker images -q | xargs -r docker rmi - -# Build -# Jobs explicitly set KUBE_FASTBUILD to desired settings. -make release - -# Push to GCS? -if [[ ${KUBE_SKIP_PUSH_GCS:-} =~ ^[yY]$ ]]; then - echo "Not pushed to GCS..." -else - readonly release_infra_clone="${WORKSPACE}/_tmp/release.git" - mkdir -p "${WORKSPACE}/_tmp" - git clone https://github.com/kubernetes/release "${release_infra_clone}" - - push_build=${release_infra_clone}/push-build.sh - - [[ -n "${KUBE_GCS_RELEASE_BUCKET-}" ]] \ - && bucket_flag="--bucket=${KUBE_GCS_RELEASE_BUCKET-}" - [[ -n "${KUBE_GCS_RELEASE_SUFFIX-}" ]] \ - && gcs_suffix_flag="--gcs-suffix=${KUBE_GCS_RELEASE_SUFFIX-}" - "${push_build}" "${bucket_flag-}" "${gcs_suffix_flag-}" \ - --nomock --verbose --ci -fi - -sha256sum _output/release-tars/kubernetes*.tar.gz diff --git a/hack/jenkins/gotest.sh b/hack/jenkins/gotest.sh deleted file mode 100755 index 71971df4af9ed..0000000000000 --- a/hack/jenkins/gotest.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Runs the unit and integration tests, production JUnit-style XML test reports -# in ${WORKSPACE}/_artifacts. - -set -o errexit -set -o nounset -set -o pipefail -set -o xtrace - -# !!! ALERT !!! Jenkins default $HOME is /var/lib/jenkins, which is -# global across jobs. We change $HOME instead to ${WORKSPACE}, which -# is an incoming variable Jenkins provides us for this job's scratch -# space. -export HOME=${WORKSPACE} # Nothing should want Jenkins $HOME -export GOPATH=${HOME}/_gopath -export PATH=${GOPATH}/bin:${HOME}/third_party/etcd:/usr/local/go/bin:${PATH} - -# Install a few things needed by unit and /integration tests. -command -v etcd &>/dev/null || ./hack/install-etcd.sh -go install k8s.io/kubernetes/vendor/gotest.tools/gotestsum - -# Enable the Go race detector. -export KUBE_RACE=-race -# Set artifacts directory -export ARTIFACTS=${ARTIFACTS:-"${WORKSPACE}/artifacts"} -# Produce a JUnit-style XML test report -export KUBE_JUNIT_REPORT_DIR="${ARTIFACTS}" -# Save the verbose stdout as well. -export KUBE_KEEP_VERBOSE_TEST_OUTPUT=y - -make test -make test-integration diff --git a/hack/jenkins/upload-to-gcs.sh b/hack/jenkins/upload-to-gcs.sh deleted file mode 100755 index 17c1415f42179..0000000000000 --- a/hack/jenkins/upload-to-gcs.sh +++ /dev/null @@ -1,296 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script uploads metadata and test results to Google Cloud Storage, in the -# location indicated by JENKINS_GCS_LOGS_PATH. By default, we use the Google -# kubernetes-jenkins bucket. -# -# The script looks for one of two environment variables to be set: -# JENKINS_BUILD_STARTED: set to a nonempty string to upload version -# information to 'started.json'. The value of the variable is not -# currently used. -# JENKINS_BUILD_FINISHED: set to the Jenkins build result to upload the build -# result to 'finished.json', any test artifacts, and update the -# 'latest-build.txt' file pointer. Since this script uses gsutil directly, -# it's a bit faster at uploading large numbers of files than the GCS Jenkins -# plugin. It also makes use of gsutil's gzip functionality. -# -# Note: for magicfile support to work correctly, the "file" utility must be -# installed. - -# TODO(rmmh): rewrite this script in Python so we can actually test it! - -set -o errexit -set -o nounset -set -o pipefail - -if [[ -n "${JENKINS_BUILD_STARTED:-}" && -n "${JENKINS_BUILD_FINISHED:-}" ]]; then - echo "Error: JENKINS_BUILD_STARTED and JENKINS_BUILD_FINISHED should not both be set!" - exit 1 -fi - -if [[ ! ${JENKINS_UPLOAD_TO_GCS:-y} =~ ^[yY]$ ]]; then - exit 0 -fi - -# Attempt to determine if we're running against a repo other than -# kubernetes/kubernetes to determine whether to place PR logs in a different -# location. -# -# In the current CI system, the tracked repo is named remote. This is not true -# in general for most devs, where origin and upstream are more common. -GCS_SUBDIR="" -readonly remote_git_repo=$(git config --get remote.remote.url | sed 's:.*github.com/::' || true) -if [[ -n "${remote_git_repo}" ]]; then - case "${remote_git_repo}" in - # main repo: nothing extra - kubernetes/kubernetes) GCS_SUBDIR="" ;; - # a different repo on the k8s org: just the repo name (strip kubernetes/) - kubernetes/*) GCS_SUBDIR="${remote_git_repo#kubernetes/}/" ;; - # any other repo: ${org}_${repo} (replace / with _) - *) GCS_SUBDIR="${remote_git_repo/\//_}/" ;; - esac - if [[ "${remote_git_repo}" != "kubernetes/kubernetes" ]]; then - # also store the repo in started.json, so Gubernator can link it properly. - export BUILD_METADATA_REPO="${remote_git_repo}" - fi -fi - -if [[ ${JOB_NAME} =~ -pull- ]]; then - : "${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/pr-logs/pull/${GCS_SUBDIR}${ghprbPullId:-unknown}"}" - : "${JENKINS_GCS_LATEST_PATH:="gs://kubernetes-jenkins/pr-logs/directory"}" - : "${JENKINS_GCS_LOGS_INDIRECT:="gs://kubernetes-jenkins/pr-logs/directory/${JOB_NAME}"}" -else - : "${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/logs"}" - : "${JENKINS_GCS_LATEST_PATH:="gs://kubernetes-jenkins/logs"}" - : "${JENKINS_GCS_LOGS_INDIRECT:=""}" -fi - -readonly artifacts_path="${WORKSPACE}/_artifacts" -readonly gcs_job_path="${JENKINS_GCS_LOGS_PATH}/${JOB_NAME}" -readonly gcs_build_path="${gcs_job_path}/${BUILD_NUMBER}" -readonly gcs_latest_path="${JENKINS_GCS_LATEST_PATH}/${JOB_NAME}" -readonly gcs_indirect_path="${JENKINS_GCS_LOGS_INDIRECT}" -readonly gcs_acl="public-read" -readonly results_url=${gcs_build_path//"gs:/"/"https://console.cloud.google.com/storage/browser"} -readonly timestamp=$(date +%s) - -######################################################################### -# $0 is called from different contexts so figure out where kubernetes is. -# Sets non-exported global kubernetes_base_path and defaults to "." -function set_kubernetes_base_path () { - for kubernetes_base_path in kubernetes go/src/k8s.io/kubernetes .; do - # Pick a canonical item to find in a kubernetes tree which could be a - # raw source tree or an expanded tarball. - - [[ -f ${kubernetes_base_path}/cluster/common.sh ]] && break - done -} - -######################################################################### -# Try to discover the kubernetes version. -# prints version -function find_version() { - ( - # Where are we? - # This could be set in the global scope at some point if we need to - # discover the kubernetes path elsewhere. - set_kubernetes_base_path - - cd ${kubernetes_base_path} - - if [[ -e "version" ]]; then - cat version - elif [[ -e "hack/lib/version.sh" ]]; then - export KUBE_ROOT="." - source "hack/lib/version.sh" - kube::version::get_version_vars - echo "${KUBE_GIT_VERSION-}" - else - # Last resort from the started.json - gsutil cat "${gcs_build_path}/started.json" 2>/dev/null |\ - sed -n 's/ *"version": *"\([^"]*\)",*/\1/p' - fi - ) -} - -# Output started.json. Use test function below! -function print_started() { - local metadata_keys - metadata_keys=$(compgen -e | grep ^BUILD_METADATA_) - echo "{" - echo " \"version\": \"${version}\"," # TODO(fejta): retire - echo " \"job-version\": \"${version}\"," - echo " \"timestamp\": ${timestamp}," - if [[ -n "${metadata_keys}" ]]; then - # Any exported variables of the form BUILD_METADATA_KEY=VALUE - # will be available as started["metadata"][KEY.lower()]. - echo " \"metadata\": {" - local sep="" # leading commas are easy to track - for env_var in ${metadata_keys}; do - local var_upper="${env_var#BUILD_METADATA_}" - echo " ${sep}\"${var_upper,,}\": \"${!env_var}\"" - sep="," - done - echo " }," - fi - echo " \"jenkins-node\": \"${NODE_NAME:-}\"" - echo "}" -} - -# Use this to test changes to print_started. -if [[ -n "${TEST_STARTED_JSON:-}" ]]; then - version=$(find_version) - jq . <(print_started) - exit -fi - -function upload_version() { - local -r version=$(find_version) - local upload_attempt - - echo -n 'Run starting at '; date -d "@${timestamp}" - - if [[ -n "${version}" ]]; then - echo "Found Kubernetes version: ${version}" - else - echo "Could not find Kubernetes version" - fi - - local -r json_file="${gcs_build_path}/started.json" - for upload_attempt in {1..3}; do - echo "Uploading version to: ${json_file} (attempt ${upload_attempt})" - gsutil -q -h "Content-Type:application/json" cp -a "${gcs_acl}" <(print_started) "${json_file}" || continue - break - done -} - -######################################################################### -# Maintain a single file storing the full build version, Jenkins' job number -# build state. Limit its size so it does not grow unbounded. -# This is primarily used for and by the -# github.com/kubernetes/release/find_green_build tool. -# @param build_result - the state of the build -# -function update_job_result_cache() { - local -r build_result=$1 - local -r version=$(find_version) - local -r job_results=${gcs_job_path}/jobResultsCache.json - local -r tmp_results="${WORKSPACE}/_tmp/jobResultsCache.tmp" - # TODO: This constraint is insufficient. The boundary for secondary - # job cache should be date based on the last primary build. - # The issue is we are trying to find a matched green set of results - # at a given hash, but all of the jobs run at wildly different lengths. - local -r cache_size=300 - local upload_attempt - - if [[ -n "${version}" ]]; then - echo "Found Kubernetes version: ${version}" - else - echo "Could not find Kubernetes version" - fi - - mkdir -p "${tmp_results%/*}" - - # Construct a valid json file - echo "[" > "${tmp_results}" - - for upload_attempt in $(seq 3); do - echo "Copying ${job_results} to ${tmp_results} (attempt ${upload_attempt})" - # The sed construct below is stripping out only the "version" lines - # and then ensuring there's a single comma at the end of the line. - gsutil -q cat "${job_results}" 2>&- |\ - sed -n 's/^\({"version".*}\),*/\1,/p' |\ - tail -${cache_size} >> "${tmp_results}" || continue - break - done - - echo "{\"version\": \"${version}\", \"buildnumber\": \"${BUILD_NUMBER}\"," \ - "\"result\": \"${build_result}\"}" >> "${tmp_results}" - - echo "]" >> "${tmp_results}" - - for upload_attempt in $(seq 3); do - echo "Copying ${tmp_results} to ${job_results} (attempt ${upload_attempt})" - gsutil -q -h "Content-Type:application/json" cp -a "${gcs_acl}" \ - "${tmp_results}" "${job_results}" || continue - break - done - - rm -f "${tmp_results}" -} - -function upload_artifacts_and_build_result() { - local -r build_result=$1 - local upload_attempt - - echo -n 'Run finished at '; date -d "@${timestamp}" - - for upload_attempt in {1..3}; do - echo "Uploading to ${gcs_build_path} (attempt ${upload_attempt})" - echo "Uploading build result: ${build_result}" - gsutil -q -h "Content-Type:application/json" cp -a "${gcs_acl}" <( - echo "{" - echo " \"result\": \"${build_result}\"," - echo " \"timestamp\": ${timestamp}" - echo "}" - ) "${gcs_build_path}/finished.json" || continue - if [[ -d "${artifacts_path}" && -n $(ls -A "${artifacts_path}") ]]; then - echo "Uploading artifacts" - gsutil -m -q -o "GSUtil:use_magicfile=True" cp -a "${gcs_acl}" -r -c \ - -z log,txt,xml "${artifacts_path}" "${gcs_build_path}/artifacts" || continue - fi - if [[ -e "${WORKSPACE}/build-log.txt" ]]; then - echo "Uploading build log" - gsutil -q cp -Z -a "${gcs_acl}" "${WORKSPACE}/build-log.txt" "${gcs_build_path}" - fi - - # For pull jobs, keep a canonical ordering for tools that want to examine - # the output. - if [[ "${gcs_indirect_path}" != "" ]]; then - echo "Writing ${gcs_build_path} to ${gcs_indirect_path}/${BUILD_NUMBER}.txt" - echo "${gcs_build_path}" | \ - gsutil -q -h "Content-Type:text/plain" \ - cp -a "${gcs_acl}" - "${gcs_indirect_path}/${BUILD_NUMBER}.txt" || continue - echo "Marking build ${BUILD_NUMBER} as the latest completed build for this PR" - echo "${BUILD_NUMBER}" | \ - gsutil -q -h "Content-Type:text/plain" -h "Cache-Control:private, max-age=0, no-transform" \ - cp -a "${gcs_acl}" - "${gcs_job_path}/latest-build.txt" || continue - fi - - # Mark this build as the latest completed. - echo "Marking build ${BUILD_NUMBER} as the latest completed build" - echo "${BUILD_NUMBER}" | \ - gsutil -q -h "Content-Type:text/plain" -h "Cache-Control:private, max-age=0, no-transform" \ - cp -a "${gcs_acl}" - "${gcs_latest_path}/latest-build.txt" || continue - break # all uploads succeeded if we hit this point - done - - echo -e "\n\n\n*** View logs and artifacts at ${results_url} ***\n\n" -} - -if [[ -z "${BOOTSTRAP_MIGRATION:-}" ]]; then - if [[ -n "${JENKINS_BUILD_STARTED:-}" ]]; then - upload_version - elif [[ -n "${JENKINS_BUILD_FINISHED:-}" ]]; then - upload_artifacts_and_build_result "${JENKINS_BUILD_FINISHED}" - update_job_result_cache "${JENKINS_BUILD_FINISHED}" - else - echo "ERROR: Called without JENKINS_BUILD_STARTED or JENKINS_BUILD_FINISHED set." - echo "ERROR: this should not happen" - exit 1 - fi -fi diff --git a/hack/jenkins/verify.sh b/hack/jenkins/verify.sh deleted file mode 100755 index 2da874fde2c1d..0000000000000 --- a/hack/jenkins/verify.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Sets up the environment (e.g. installing etcd if necessary) -# and then runs all of the verification checks. - -set -o errexit -set -o nounset -set -o pipefail -set -o xtrace - -# !!! ALERT !!! Jenkins default $HOME is /var/lib/jenkins, which is -# global across jobs. We change $HOME instead to ${WORKSPACE}, which -# is an incoming variable Jenkins provides us for this job's scratch -# space. -export HOME=${WORKSPACE} # Nothing should want Jenkins $HOME -export GOPATH=${HOME}/_gopath -export PATH=${GOPATH}/bin:${HOME}/third_party/etcd:/usr/local/go/bin:${PATH} - -# Install a few things needed by the verification tests. -command -v etcd &>/dev/null || ./hack/install-etcd.sh - -make verify