From 5ea67d911f79559939846cfe6e69bfe29648fdfc Mon Sep 17 00:00:00 2001 From: husharp Date: Tue, 5 Dec 2023 11:54:27 +0800 Subject: [PATCH 1/8] add Signed-off-by: husharp --- ...pod-pull_integration_realcluster_test.yaml | 46 +++++++++++++ .../pull_integration_realcluster_test.groovy | 67 +++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 pipelines/tikv/pd/latest/pod-pull_integration_realcluster_test.yaml create mode 100644 pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy diff --git a/pipelines/tikv/pd/latest/pod-pull_integration_realcluster_test.yaml b/pipelines/tikv/pd/latest/pod-pull_integration_realcluster_test.yaml new file mode 100644 index 0000000000..62c0fdbf33 --- /dev/null +++ b/pipelines/tikv/pd/latest/pod-pull_integration_realcluster_test.yaml @@ -0,0 +1,46 @@ +apiVersion: v1 +kind: Pod +spec: + securityContext: + fsGroup: 1000 + containers: + - name: golang + image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" + tty: true + resources: + requests: + memory: 12Gi + cpu: "4" + limits: + memory: 16Gi + cpu: "6" + - name: net-tool + image: wbitt/network-multitool + tty: true + resources: + limits: + memory: 128Mi + cpu: 100m + - name: report + image: hub.pingcap.net/jenkins/python3-requests:latest + tty: true + resources: + limits: + memory: 256Mi + cpu: 100m + - name: status-updater + image: hub.pingcap.net/pingcap/github-status-updater:latest + tty: true + resources: + limits: + memory: 256Mi + cpu: 100m + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 diff --git a/pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy b/pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy new file mode 100644 index 0000000000..48f8beb773 --- /dev/null +++ b/pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy @@ -0,0 +1,67 @@ +// REF: https://www.jenkins.io/doc/book/pipeline/syntax/#declarative-pipeline +// Keep small than 400 lines: https://issues.jenkins.io/browse/JENKINS-37984 +@Library('tipipeline') _ + +final K8S_NAMESPACE = "jenkins-pd" +final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh' +final GIT_FULL_REPO_NAME = 'tikv/pd' +final POD_TEMPLATE_FILE = 'pipelines/tikv/pd/latest/pod-pull_integration_realcluster_test.yaml' +final REFS = readJSON(text: params.JOB_SPEC).refs + +pipeline { + agent { + kubernetes { + namespace K8S_NAMESPACE + yamlFile POD_TEMPLATE_FILE + defaultContainer 'golang' + } + } + environment { + FILE_SERVER_URL = 'http://fileserver.pingcap.net' + } + options { + timeout(time: 40, unit: 'MINUTES') + parallelsAlwaysFailFast() + } + stages { + stage('Debug info') { + steps { + sh label: 'Debug info', script: """ + printenv + echo "-------------------------" + go env + echo "-------------------------" + echo "debug command: kubectl -n ${K8S_NAMESPACE} exec -ti ${NODE_NAME} bash" + """ + container(name: 'net-tool') { + sh 'dig github.com' + } + } + } + stage('Checkout') { + options { timeout(time: 10, unit: 'MINUTES') } + steps { + dir("pd") { + cache(path: "./", includes: '**/*', key: prow.getCacheKey('git', REFS), restoreKeys: prow.getRestoreKeys('git', REFS)) { + retry(2) { + script { + prow.checkoutRefs(REFS) + } + } + } + } + } + } + stage('Tests') { + options { timeout(time: 20, unit: 'MINUTES') } + steps { + dir('pd/') { + sh label: "PD Real Cluster Check", script: """ + cd tests/integrations/realtiup + make check + """ + } + } + } + } +} From 658bf5ff0866a6caed68de783eecaf1ec65bce13 Mon Sep 17 00:00:00 2001 From: husharp Date: Tue, 5 Dec 2023 12:46:39 +0800 Subject: [PATCH 2/8] add jobs Signed-off-by: husharp --- .../pull_integration_realcluster_test.groovy | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 jobs/tikv/pd/latest/pull_integration_realcluster_test.groovy diff --git a/jobs/tikv/pd/latest/pull_integration_realcluster_test.groovy b/jobs/tikv/pd/latest/pull_integration_realcluster_test.groovy new file mode 100644 index 0000000000..5a342facff --- /dev/null +++ b/jobs/tikv/pd/latest/pull_integration_realcluster_test.groovy @@ -0,0 +1,39 @@ +// REF: https:///plugin/job-dsl/api-viewer/index.html +// For trunk and latest release branches. +pipelineJob('tikv/pd/pull_integration_realcluster_test') { + logRotator { + daysToKeep(30) + } + parameters { + // Ref: https://docs.prow.k8s.io/docs/jobs/#job-environment-variables + stringParam("BUILD_ID") + stringParam("PROW_JOB_ID") + stringParam("JOB_SPEC", "", "Prow job spec struct data") + } + properties { + buildFailureAnalyzer(false) // disable failure analyze + githubProjectUrl("https://github.com/tikv/pd") + } + + definition { + cpsScm { + lightweight(true) + scriptPath("pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy") + scm { + git{ + remote { + url('https://github.com/PingCAP-QE/ci.git') + } + branch('main') + extensions { + cloneOptions { + depth(1) + shallow(true) + timeout(5) + } + } + } + } + } + } +} From ac98834241a9ef720527d539d116a2d132346b97 Mon Sep 17 00:00:00 2001 From: wuhuizuo Date: Tue, 5 Dec 2023 13:41:53 +0800 Subject: [PATCH 3/8] Update pipelines/tikv/pd/latest/pod-pull_integration_realcluster_test.yaml --- .../pod-pull_integration_realcluster_test.yaml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/pipelines/tikv/pd/latest/pod-pull_integration_realcluster_test.yaml b/pipelines/tikv/pd/latest/pod-pull_integration_realcluster_test.yaml index 62c0fdbf33..4c5d89c172 100644 --- a/pipelines/tikv/pd/latest/pod-pull_integration_realcluster_test.yaml +++ b/pipelines/tikv/pd/latest/pod-pull_integration_realcluster_test.yaml @@ -21,20 +21,6 @@ spec: limits: memory: 128Mi cpu: 100m - - name: report - image: hub.pingcap.net/jenkins/python3-requests:latest - tty: true - resources: - limits: - memory: 256Mi - cpu: 100m - - name: status-updater - image: hub.pingcap.net/pingcap/github-status-updater:latest - tty: true - resources: - limits: - memory: 256Mi - cpu: 100m affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: From 85e7bf823a4067bb70446ff3166c04cc9729a44c Mon Sep 17 00:00:00 2001 From: wuhuizuo Date: Tue, 5 Dec 2023 13:42:11 +0800 Subject: [PATCH 4/8] Update jobs/tikv/pd/latest/pull_integration_realcluster_test.groovy --- jobs/tikv/pd/latest/pull_integration_realcluster_test.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/tikv/pd/latest/pull_integration_realcluster_test.groovy b/jobs/tikv/pd/latest/pull_integration_realcluster_test.groovy index 5a342facff..fbc80a6b29 100644 --- a/jobs/tikv/pd/latest/pull_integration_realcluster_test.groovy +++ b/jobs/tikv/pd/latest/pull_integration_realcluster_test.groovy @@ -11,7 +11,7 @@ pipelineJob('tikv/pd/pull_integration_realcluster_test') { stringParam("JOB_SPEC", "", "Prow job spec struct data") } properties { - buildFailureAnalyzer(false) // disable failure analyze + buildFailureAnalyzer(false) githubProjectUrl("https://github.com/tikv/pd") } From e150b05bb0f5b55be858f46ab1b2f82b950bd6da Mon Sep 17 00:00:00 2001 From: wuhuizuo Date: Tue, 5 Dec 2023 13:44:31 +0800 Subject: [PATCH 5/8] Update pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy --- .../tikv/pd/latest/pull_integration_realcluster_test.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy b/pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy index 48f8beb773..3067772ff9 100644 --- a/pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy +++ b/pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy @@ -3,7 +3,6 @@ @Library('tipipeline') _ final K8S_NAMESPACE = "jenkins-pd" -final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh' final GIT_FULL_REPO_NAME = 'tikv/pd' final POD_TEMPLATE_FILE = 'pipelines/tikv/pd/latest/pod-pull_integration_realcluster_test.yaml' final REFS = readJSON(text: params.JOB_SPEC).refs From b5b75fa75084e5125df43772f890e69d7df54b01 Mon Sep 17 00:00:00 2001 From: wuhuizuo Date: Tue, 5 Dec 2023 13:44:51 +0800 Subject: [PATCH 6/8] Update pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy --- .../tikv/pd/latest/pull_integration_realcluster_test.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy b/pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy index 3067772ff9..1d5327c812 100644 --- a/pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy +++ b/pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy @@ -3,7 +3,6 @@ @Library('tipipeline') _ final K8S_NAMESPACE = "jenkins-pd" -final GIT_FULL_REPO_NAME = 'tikv/pd' final POD_TEMPLATE_FILE = 'pipelines/tikv/pd/latest/pod-pull_integration_realcluster_test.yaml' final REFS = readJSON(text: params.JOB_SPEC).refs From 4e10b00a59f6bc7ed28d62cc0f4e1c2827b4bd0f Mon Sep 17 00:00:00 2001 From: Purelind Date: Thu, 7 Dec 2023 15:25:30 +0800 Subject: [PATCH 7/8] Update pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy --- .../tikv/pd/latest/pull_integration_realcluster_test.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy b/pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy index 1d5327c812..d54c4ad4af 100644 --- a/pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy +++ b/pipelines/tikv/pd/latest/pull_integration_realcluster_test.groovy @@ -53,7 +53,7 @@ pipeline { stage('Tests') { options { timeout(time: 20, unit: 'MINUTES') } steps { - dir('pd/') { + dir('pd') { sh label: "PD Real Cluster Check", script: """ cd tests/integrations/realtiup make check From 76b9e6ec67876994aef1279388e8e7f1a9f50186 Mon Sep 17 00:00:00 2001 From: Purelind Date: Thu, 7 Dec 2023 15:25:36 +0800 Subject: [PATCH 8/8] Update pipelines/tikv/pd/latest/pod-pull_integration_realcluster_test.yaml --- .../tikv/pd/latest/pod-pull_integration_realcluster_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipelines/tikv/pd/latest/pod-pull_integration_realcluster_test.yaml b/pipelines/tikv/pd/latest/pod-pull_integration_realcluster_test.yaml index 4c5d89c172..1e88914c54 100644 --- a/pipelines/tikv/pd/latest/pod-pull_integration_realcluster_test.yaml +++ b/pipelines/tikv/pd/latest/pod-pull_integration_realcluster_test.yaml @@ -9,8 +9,8 @@ spec: tty: true resources: requests: - memory: 12Gi - cpu: "4" + memory: 16Gi + cpu: "6" limits: memory: 16Gi cpu: "6"