Tweak promotion pipeline to support rpm and qualifier (#2094)

* Tweak promotion pipeline to support rpm and qualifier Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Add qualifier tests Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Add signer tests Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Update core plugins path after Sarat PR merged opensearch-project/OpenSearch#3193 Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Remove extra comments Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Rename promoteTarArtifacts promoteArtifacts Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>
opensearch-project · May 11, 2022 · f318149 · f318149
1 parent f3b52c4
commit f318149
Show file tree

Hide file tree

Showing 41 changed files with 6,009 additions and 249 deletions.
diff --git a/jenkins/check-for-build.jenkinsfile b/jenkins/check-for-build.jenkinsfile
@@ -2,6 +2,7 @@ lib = library(identifier: "jenkins@20211123", retriever: legacySCM(scm))
 
 pipeline {
     options {
+        timeout(time: 1, unit: 'HOURS')
         buildDiscarder(logRotator(daysToKeepStr: '7'))
     }
     agent none

diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile
@@ -2,7 +2,7 @@ lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm))
 
 pipeline {
     options {
-        timeout(time: 3, unit: 'HOURS')
+        timeout(time: 4, unit: 'HOURS')
         copyArtifactPermission("${JOB_BASE_NAME}")
         buildDiscarder(logRotator(artifactNumToKeepStr: '1'))
     }

diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile
@@ -1,6 +1,9 @@
 lib = library(identifier: "jenkins@20211123", retriever: legacySCM(scm))
 
 pipeline {
+    options {
+        timeout(time: 4, unit: 'HOURS')
+    }
     agent none
     environment {
         AGENT_X64 = 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host'

diff --git a/jenkins/promotion/tar-promotion.jenkinsfile → jenkins/promotion/promotion.jenkinsfile b/jenkins/promotion/tar-promotion.jenkinsfile → jenkins/promotion/promotion.jenkinsfile
@@ -45,7 +45,8 @@ pipeline {
             }
             steps {
                 script {
-                    promoteTarArtifacts(
+                    currentBuild.description = "${DISTRIBUTION_BUILD_NUMBER} ${DISTRIBUTION_PLATFORM} ${DISTRIBUTION_ARCHITECTURE} ${INPUT_MANIFEST}"
+                    promoteArtifacts(
                         fileActions: [createSha512Checksums(), createSignatureFiles()]
                     )
                 }

diff --git a/src/sign_workflow/signer.py b/src/sign_workflow/signer.py
@@ -18,7 +18,7 @@
 
 
 class Signer:
-    ACCEPTED_FILE_TYPES = [".zip", ".jar", ".war", ".pom", ".module", ".tar.gz", ".whl", ".crate"]
+    ACCEPTED_FILE_TYPES = [".zip", ".jar", ".war", ".pom", ".module", ".tar.gz", ".whl", ".crate", ".rpm"]
 
     def __init__(self):
         self.git_repo = GitRepository(self.get_repo_url(), "HEAD", working_subdirectory="src")

diff --git a/tests/data/opensearch-build-1.3.0-rpm.yml b/tests/data/opensearch-build-1.3.0-rpm.yml
@@ -9,7 +9,7 @@ build:
 components:
   - artifacts:
       dist:
-        - dist/opensearch-min-1.3.0-linux-x64.tar.gz
+        - dist/opensearch-min-1.3.0-linux-x64.rpm
       core-plugins:
         - core-plugins/discovery-ec2-1.3.0.zip
         - core-plugins/transport-nio-1.3.0.zip

diff --git a/tests/data/opensearch-build-2.0.0-rc1-rpm.yml b/tests/data/opensearch-build-2.0.0-rc1-rpm.yml
diff --git a/tests/data/opensearch-build-2.0.0-rc1.yml b/tests/data/opensearch-build-2.0.0-rc1.yml
diff --git a/tests/data/opensearch-dashboards-build-1.3.0-rpm.yml b/tests/data/opensearch-dashboards-build-1.3.0-rpm.yml
@@ -0,0 +1,87 @@
+---
+build:
+  architecture: x64
+  id: 25b38c278cdd45efa583765d8ba76346
+  name: OpenSearch Dashboards
+  platform: linux
+  version: 1.3.0
+  distribution: rpm
+components:
+  - artifacts:
+      dist:
+        - dist/opensearch-dashboards-min-1.3.0-linux-x64.rpm
+    commit_id: caf668e73304bac890f41c37cd6c3a41257cd289
+    name: OpenSearch-Dashboards
+    ref: tags/1.3.0
+    repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/alertingDashboards-1.3.0.zip
+    commit_id: 07d4cb8bc9c62b15bf7cb4fd4e3fc3a818f9152d
+    name: alertingDashboards
+    ref: tags/1.3.0.0
+    repository: https://github.com/opensearch-project/alerting-dashboards-plugin
+    version: 1.3.0.0
+  - commit_id: 208227256bf6dece2976da528bf41bae56e4effc
+    name: functionalTestDashboards
+    ref: main
+    repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/securityDashboards-1.3.0.zip
+    commit_id: 960ef753f74dceb793e8936a40ac8c6cbec88dd1
+    name: securityDashboards
+    ref: '1.3'
+    repository: https://github.com/opensearch-project/security-dashboards-plugin.git
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/indexManagementDashboards-1.3.0.zip
+    commit_id: 6748b18ee172138b4811a459053f4a43c1486821
+    name: indexManagementDashboards
+    ref: '1.3'
+    repository: https://github.com/opensearch-project/index-management-dashboards-plugin
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/queryWorkbenchDashboards-1.3.0.zip
+    commit_id: a675eac2ed32a228184c47c289cfd22fde479139
+    name: queryWorkbenchDashboards
+    ref: 1.3.0.0
+    repository: https://github.com/opensearch-project/sql.git
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/reportsDashboards-1.3.0.zip
+    commit_id: 290523a6d825c9c073818aebfcbe4813cb2554bc
+    name: reportsDashboards
+    ref: 1.3.0.0
+    repository: https://github.com/opensearch-project/dashboards-reports.git
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/observabilityDashboards-1.3.0.zip
+    commit_id: 4fde4040c0a43233b41a862a169e2bd1616f6b1b
+    name: observabilityDashboards
+    ref: 1.3.0.0
+    repository: https://github.com/opensearch-project/trace-analytics.git
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/ganttChartDashboards-1.3.0.zip
+    commit_id: 2123bc65873e026cb8b0fd26816a6b1980bd19ce
+    name: ganttChartDashboards
+    ref: 1.3.0.0
+    repository: https://github.com/opensearch-project/dashboards-visualizations.git
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/anomalyDetectionDashboards-1.3.0.zip
+    commit_id: c4921f4471aeab38db80eb47cd92b4f7a3e04126
+    name: anomalyDetectionDashboards
+    ref: tags/1.3.0.0
+    repository: https://github.com/opensearch-project/anomaly-detection-dashboards-plugin
+    version: 1.3.0.0
+schema-version: '1.2'
diff --git a/tests/data/opensearch-dashboards-build-1.3.0.yml b/tests/data/opensearch-dashboards-build-1.3.0.yml
@@ -0,0 +1,87 @@
+---
+build:
+  architecture: x64
+  id: 25b38c278cdd45efa583765d8ba76346
+  name: OpenSearch Dashboards
+  platform: linux
+  version: 1.3.0
+  distribution: tar
+components:
+  - artifacts:
+      dist:
+        - dist/opensearch-dashboards-min-1.3.0-linux-x64.tar.gz
+    commit_id: caf668e73304bac890f41c37cd6c3a41257cd289
+    name: OpenSearch-Dashboards
+    ref: tags/1.3.0
+    repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/alertingDashboards-1.3.0.zip
+    commit_id: 07d4cb8bc9c62b15bf7cb4fd4e3fc3a818f9152d
+    name: alertingDashboards
+    ref: tags/1.3.0.0
+    repository: https://github.com/opensearch-project/alerting-dashboards-plugin
+    version: 1.3.0.0
+  - commit_id: 208227256bf6dece2976da528bf41bae56e4effc
+    name: functionalTestDashboards
+    ref: main
+    repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/securityDashboards-1.3.0.zip
+    commit_id: 960ef753f74dceb793e8936a40ac8c6cbec88dd1
+    name: securityDashboards
+    ref: '1.3'
+    repository: https://github.com/opensearch-project/security-dashboards-plugin.git
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/indexManagementDashboards-1.3.0.zip
+    commit_id: 6748b18ee172138b4811a459053f4a43c1486821
+    name: indexManagementDashboards
+    ref: '1.3'
+    repository: https://github.com/opensearch-project/index-management-dashboards-plugin
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/queryWorkbenchDashboards-1.3.0.zip
+    commit_id: a675eac2ed32a228184c47c289cfd22fde479139
+    name: queryWorkbenchDashboards
+    ref: 1.3.0.0
+    repository: https://github.com/opensearch-project/sql.git
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/reportsDashboards-1.3.0.zip
+    commit_id: 290523a6d825c9c073818aebfcbe4813cb2554bc
+    name: reportsDashboards
+    ref: 1.3.0.0
+    repository: https://github.com/opensearch-project/dashboards-reports.git
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/observabilityDashboards-1.3.0.zip
+    commit_id: 4fde4040c0a43233b41a862a169e2bd1616f6b1b
+    name: observabilityDashboards
+    ref: 1.3.0.0
+    repository: https://github.com/opensearch-project/trace-analytics.git
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/ganttChartDashboards-1.3.0.zip
+    commit_id: 2123bc65873e026cb8b0fd26816a6b1980bd19ce
+    name: ganttChartDashboards
+    ref: 1.3.0.0
+    repository: https://github.com/opensearch-project/dashboards-visualizations.git
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/anomalyDetectionDashboards-1.3.0.zip
+    commit_id: c4921f4471aeab38db80eb47cd92b4f7a3e04126
+    name: anomalyDetectionDashboards
+    ref: tags/1.3.0.0
+    repository: https://github.com/opensearch-project/anomaly-detection-dashboards-plugin
+    version: 1.3.0.0
+schema-version: '1.2'
diff --git a/tests/data/opensearch-dashboards-build-2.0.0-rc1-rpm.yml b/tests/data/opensearch-dashboards-build-2.0.0-rc1-rpm.yml
@@ -0,0 +1,87 @@
+---
+build:
+  architecture: x64
+  id: 25b38c278cdd45efa583765d8ba76346
+  name: OpenSearch Dashboards
+  platform: linux
+  version: 2.0.0-rc1
+  distribution: rpm
+components:
+  - artifacts:
+      dist:
+        - dist/opensearch-dashboards-min-2.0.0-rc1-linux-x64.rpm
+    commit_id: caf668e73304bac890f41c37cd6c3a41257cd289
+    name: OpenSearch-Dashboards
+    ref: tags/2.0.0-rc1
+    repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git
+    version: 2.0.0.0-rc1
+  - artifacts:
+      plugins:
+        - plugins/alertingDashboards-2.0.0-rc1.zip
+    commit_id: 07d4cb8bc9c62b15bf7cb4fd4e3fc3a818f9152d
+    name: alertingDashboards
+    ref: tags/2.0.0.0-rc1
+    repository: https://github.com/opensearch-project/alerting-dashboards-plugin
+    version: 2.0.0.0-rc1
+  - commit_id: 208227256bf6dece2976da528bf41bae56e4effc
+    name: functionalTestDashboards
+    ref: main
+    repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git
+    version: 2.0.0.0-rc1
+  - artifacts:
+      plugins:
+        - plugins/securityDashboards-2.0.0-rc1.zip
+    commit_id: 960ef753f74dceb793e8936a40ac8c6cbec88dd1
+    name: securityDashboards
+    ref: '2.0'
+    repository: https://github.com/opensearch-project/security-dashboards-plugin.git
+    version: 2.0.0.0-rc1
+  - artifacts:
+      plugins:
+        - plugins/indexManagementDashboards-2.0.0-rc1.zip
+    commit_id: 6748b18ee172138b4811a459053f4a43c1486821
+    name: indexManagementDashboards
+    ref: '2.0'
+    repository: https://github.com/opensearch-project/index-management-dashboards-plugin
+    version: 2.0.0.0-rc1
+  - artifacts:
+      plugins:
+        - plugins/queryWorkbenchDashboards-2.0.0-rc1.zip
+    commit_id: a675eac2ed32a228184c47c289cfd22fde479139
+    name: queryWorkbenchDashboards
+    ref: 2.0.0.0-rc1
+    repository: https://github.com/opensearch-project/sql.git
+    version: 2.0.0.0-rc1
+  - artifacts:
+      plugins:
+        - plugins/reportsDashboards-2.0.0-rc1.zip
+    commit_id: 290523a6d825c9c073818aebfcbe4813cb2554bc
+    name: reportsDashboards
+    ref: 2.0.0.0-rc1
+    repository: https://github.com/opensearch-project/dashboards-reports.git
+    version: 2.0.0.0-rc1
+  - artifacts:
+      plugins:
+        - plugins/observabilityDashboards-2.0.0-rc1.zip
+    commit_id: 4fde4040c0a43233b41a862a169e2bd1616f6b1b
+    name: observabilityDashboards
+    ref: 2.0.0.0-rc1
+    repository: https://github.com/opensearch-project/trace-analytics.git
+    version: 2.0.0.0-rc1
+  - artifacts:
+      plugins:
+        - plugins/ganttChartDashboards-2.0.0-rc1.zip
+    commit_id: 2123bc65873e026cb8b0fd26816a6b1980bd19ce
+    name: ganttChartDashboards
+    ref: 2.0.0.0-rc1
+    repository: https://github.com/opensearch-project/dashboards-visualizations.git
+    version: 2.0.0.0-rc1
+  - artifacts:
+      plugins:
+        - plugins/anomalyDetectionDashboards-2.0.0-rc1.zip
+    commit_id: c4921f4471aeab38db80eb47cd92b4f7a3e04126
+    name: anomalyDetectionDashboards
+    ref: tags/2.0.0.0-rc1
+    repository: https://github.com/opensearch-project/anomaly-detection-dashboards-plugin
+    version: 2.0.0.0-rc1
+schema-version: '1.2'