Merge main into oss-docs and fix related bootstrap and test failures (#…

…137)

* Remove any non oss from build, package, and distribution (#102)

This commit changes the building, packaging, and testing framework to only support OSS on different distributions. 

Next steps:

completely remove -oss flag dependencies in package and build tests
move 6.x bwc testing to be an explicit option
remove any references to elastic.co download site (or replace with downloads from the OSS website)

Co-authored-by: Himanshu Setia <setiah@amazon.com>
Co-authored-by: Rabi Panda <pandarab@amazon.com>
Co-authored-by: Himanshu Setia <58999915+setiah@users.noreply.github.com>
Co-authored-by: Sarat Vemulapalli <vemsarat@amazon.com>

* Remove x-pact from RESI API username and password (#117)

* Update signoff message (#121)

Signed-off-by: Harold Wang <harowang@amazon.com>

* Update CI workflow to work on new infra (#123)

* Update CI workflow to work on new infra

- Backward compatability tests are disabled during CI by default #113
- Added property to allow for disabling bwc tests
- Added agent label to use specific hardware https://www.jenkins.io/doc/book/pipeline/syntax/#agent

Signed-off-by: Peter Nied <petern@amazon.com>

* Disable BWC checks. (#130)

As part of this PR, we are disabling the BWC checks. Once we have finalized the versions for the fork, we can re-enable it with right configurations.

Relates #105

Signed-off-by: Rabi Panda <pandarab@amazon.com>

* Adding test distribution logic to fix failures from main merge

Co-authored-by: Nick Knize <nknize@gmail.com>
Co-authored-by: Rabi Panda <pandarab@amazon.com>
Co-authored-by: Sarat Vemulapalli <vemsarat@amazon.com>
Co-authored-by: Harold Wang <74381974+harold-wang@users.noreply.github.com>
Co-authored-by: Peter Nied <petern@amazon.com>

CONTRIBUTING.md

@@ -34,6 +34,44 @@ Feature requests
 If you find yourself wishing for a feature that doesn't exist in Elasticsearch, you are probably not alone. There are bound to be others out there with similar needs. Many of the features that Elasticsearch has today have been added because our users saw the need.
 Open an issue on our [issues list](https://github.com/elastic/elasticsearch/issues) on GitHub which describes the feature you would like to see, why you need it, and how it should work.
 
+
+## Sign your work
+The sign-off is a simple line at the end of each commit, which certifies that you wrote it or otherwise have the right to pass it on as an open-source patch. if you can certify the below
+```
+By making a contribution to this project, I certify that:
+(a) The contribution was created in whole or in part by me and I
+have the right to submit it under the open source license
+indicated in the file; or
+(b) The contribution is based upon previous work that, to the best
+of my knowledge, is covered under an appropriate open source
+license and I have the right under that license to submit that
+work with modifications, whether created in whole or in part
+by me, under the same open source license (unless I am
+permitted to submit under a different license), as indicated
+in the file; or
+(c) The contribution was provided directly to me by some other
+person who certified (a), (b) or (c) and I have not modified
+it.
+(d) I understand and agree that this project and the contribution
+are public and that a record of the contribution (including all
+personal information I submit with it, including my sign-off) is
+maintained indefinitely and may be redistributed consistent with
+this project or the open source license(s) involved.
+```
+then you just add a line to every git commit message:
+```
+Signed-off-by: Bob Sanders <bob.sanders@email.com>
+```
+You can sign off your work easily by adding the configuration in github
+```
+git config user.name "Bob Sanders"
+git config user.email "bob.sanders@email.com"
+```
+Then, you could sign off commits automatically by adding `-s` or `-=signoff` parameter to your usual git commits commands. e.g.
+```
+git commit -s -m "my first commit"
+```
+
 Contributing code and documentation changes
 -------------------------------------------
 

Jenkinsfile

@@ -1,11 +1,13 @@
 pipeline {
-    agent any
+    agent { label 'search-cloud-ec2-c518xlarge' }
+
 
     stages {
         stage('Build') {
             steps {
                 echo 'Building..'
-                sh './gradlew check --no-daemon'
+                // Disable backward compability tasks
+                sh './gradlew check --no-daemon --no-scan -Pbwc_tests_enabled=false'
             }
         }
         stage('Test') {

build.gradle

@@ -170,8 +170,9 @@ tasks.register("verifyVersions") {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = false
+/* place a PR link here when committing bwc changes */
+final String bwc_tests_disabled_issue = "https://github.com/opendistro-for-elasticsearch/search/issues/105"
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

buildSrc/build.gradle

@@ -171,12 +171,8 @@ if (project != rootProject) {
 
   dependencies {
     reaper project('reaper')
-    distribution project(':distribution:archives:windows-zip')
     distribution project(':distribution:archives:oss-windows-zip')
-    distribution project(':distribution:archives:darwin-tar')
     distribution project(':distribution:archives:oss-darwin-tar')
-    distribution project(':distribution:archives:linux-aarch64-tar')
-    distribution project(':distribution:archives:linux-tar')
     distribution project(':distribution:archives:oss-linux-tar')
     distribution project(':distribution:archives:oss-linux-aarch64-tar')
 

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy

@@ -108,7 +108,7 @@ class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
 
         then:
         result.tasks.size() == 3
-        result.output.count("Unpacking elasticsearch-${version}-linux-x86_64.tar.gz " +
+        result.output.count("Unpacking elasticsearch-oss-${version}-linux-x86_64.tar.gz " +
                 "using SymbolicLinkPreservingUntarTransform.") == 1
     }
 
@@ -155,4 +155,4 @@ class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
             }
             """
     }
-}
+}

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy

@@ -112,6 +112,7 @@ abstract class AbstractGradleFuncTest extends Specification {
     }
 
     void setupLocalGitRepo() {
+        //TODO: cleanup
         execute("git init")
         execute('git config user.email "build-tool@elastic.co"')
         execute('git config user.name "Build tool"')

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/DistributionDownloadFixture.groovy

@@ -54,7 +54,7 @@ class DistributionDownloadFixture {
     private static String urlPath(String version,ElasticsearchDistribution.Platform platform) {
         String fileType = ((platform == ElasticsearchDistribution.Platform.LINUX ||
                 platform == ElasticsearchDistribution.Platform.DARWIN)) ? "tar.gz" : "zip"
-        "/downloads/elasticsearch/elasticsearch-${version}-${platform}-x86_64.$fileType"
+        "/downloads/elasticsearch/elasticsearch-oss-${version}-${platform}-x86_64.$fileType"
     }
 
     private static byte[] filebytes(String urlPath) throws IOException {

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPluginFuncTest.groovy

@@ -33,7 +33,7 @@ class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncT
     def setup() {
         buildFile << """
         import org.elasticsearch.gradle.tar.SymbolicLinkPreservingTar
-        
+
         plugins {
             id 'elasticsearch.internal-distribution-archive-setup'
         }
@@ -60,7 +60,6 @@ class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncT
 
         where:
         buildTaskName       | expectedOutputArchivePath
-        "buildDarwinTar"    | "darwin-tar/build/distributions/elasticsearch.tar.gz"
         "buildOssDarwinTar" | "oss-darwin-tar/build/distributions/elasticsearch-oss.tar.gz"
     }
 
@@ -82,7 +81,6 @@ class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncT
 
         where:
         buildTaskName       | expectedOutputArchivePath
-        "buildDarwinZip"    | "darwin-zip/build/distributions/elasticsearch.zip"
         "buildOssDarwinZip" | "oss-darwin-zip/build/distributions/elasticsearch-oss.zip"
     }
 
@@ -111,23 +109,23 @@ class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncT
                 }
             }
         }
-        
+
         project('consumer') { p ->
             configurations {
                 consumeArchive {}
                 consumeDir {}
             }
-            
+
             dependencies {
                 consumeDir project(path: ':producer-tar', configuration:'extracted')
                 consumeArchive project(path: ':producer-tar', configuration:'default' )
             }
-            
+
             tasks.register("copyDir", Copy) {
                 from(configurations.consumeDir)
                 into('build/dir')
             }
-            
+
             tasks.register("copyArchive", Copy) {
                 from(configurations.consumeArchive)
                 into('build/archives')
@@ -140,8 +138,8 @@ class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncT
         then: "tar task executed and target folder contains plain tar"
         result.task(':buildProducerTar').outcome == TaskOutcome.SUCCESS
         result.task(':consumer:copyArchive').outcome == TaskOutcome.SUCCESS
-        file("producer-tar/build/distributions/elasticsearch.tar.gz").exists()
-        file("consumer/build/archives/elasticsearch.tar.gz").exists()
+        file("producer-tar/build/distributions/elasticsearch-oss.tar.gz").exists()
+        file("consumer/build/archives/elasticsearch-oss.tar.gz").exists()
 
         when:
         result = gradleRunner("copyDir", "-Pversion=1.0").build()

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy

@@ -46,32 +46,29 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGradleFuncTest
     def "builds distribution from branches via archives assemble"() {
         when:
         def result = gradleRunner(new File(testProjectDir.root, "remote"),
-                ":distribution:bwc:bugfix:buildBwcDarwinTar",
                 ":distribution:bwc:bugfix:buildBwcOssDarwinTar",
                 "-DtestRemoteRepo=" + remoteGitRepo,
                 "-Dbwc.remote=origin")
                 .build()
         then:
-        result.task(":distribution:bwc:bugfix:buildBwcDarwinTar").outcome == TaskOutcome.SUCCESS
         result.task(":distribution:bwc:bugfix:buildBwcOssDarwinTar").outcome == TaskOutcome.SUCCESS
 
         and: "assemble task triggered"
-        result.output.contains("[8.0.1] > Task :distribution:archives:darwin-tar:assemble")
         result.output.contains("[8.0.1] > Task :distribution:archives:oss-darwin-tar:assemble")
     }
 
     def "bwc distribution archives can be resolved as bwc project artifact"() {
         setup:
         new File(testProjectDir.root, 'remote/build.gradle') << """
-        
+
         configurations {
             dists
         }
-        
+
         dependencies {
-            dists project(path: ":distribution:bwc:bugfix", configuration:"darwin-tar")
+            dists project(path: ":distribution:bwc:bugfix", configuration:"oss-darwin-tar")
         }
-        
+
         tasks.register("resolveDistributionArchive") {
             inputs.files(configurations.dists)
             doLast {
@@ -89,27 +86,27 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGradleFuncTest
                 .build()
         then:
         result.task(":resolveDistributionArchive").outcome == TaskOutcome.SUCCESS
-        result.task(":distribution:bwc:bugfix:buildBwcDarwinTar").outcome == TaskOutcome.SUCCESS
+        result.task(":distribution:bwc:bugfix:buildBwcOssDarwinTar").outcome == TaskOutcome.SUCCESS
 
         and: "assemble task triggered"
-        result.output.contains("[8.0.1] > Task :distribution:archives:darwin-tar:assemble")
+        result.output.contains("[8.0.1] > Task :distribution:archives:oss-darwin-tar:assemble")
         normalizedOutput(result.output)
-                .contains("distfile /distribution/bwc/bugfix/build/bwc/checkout-8.0/distribution/archives/darwin-tar/" +
-                        "build/distributions/elasticsearch-8.0.1-SNAPSHOT-darwin-x86_64.tar.gz")
+                .contains("distfile /distribution/bwc/bugfix/build/bwc/checkout-8.0/distribution/archives/oss-darwin-tar/" +
+                        "build/distributions/elasticsearch-oss-8.0.1-SNAPSHOT-darwin-x86_64.tar.gz")
     }
 
     def "bwc expanded distribution folder can be resolved as bwc project artifact"() {
         setup:
         new File(testProjectDir.root, 'remote/build.gradle') << """
-        
+
         configurations {
             expandedDist
         }
-        
+
         dependencies {
-            expandedDist project(path: ":distribution:bwc:bugfix", configuration:"expanded-darwin-tar")
+            expandedDist project(path: ":distribution:bwc:bugfix", configuration:"expanded-oss-darwin-tar")
         }
-        
+
         tasks.register("resolveExpandedDistribution") {
             inputs.files(configurations.expandedDist)
             doLast {
@@ -127,13 +124,13 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGradleFuncTest
                 .build()
         then:
         result.task(":resolveExpandedDistribution").outcome == TaskOutcome.SUCCESS
-        result.task(":distribution:bwc:bugfix:buildBwcDarwinTar").outcome == TaskOutcome.SUCCESS
+        result.task(":distribution:bwc:bugfix:buildBwcOssDarwinTar").outcome == TaskOutcome.SUCCESS
 
         and: "assemble task triggered"
-        result.output.contains("[8.0.1] > Task :distribution:archives:darwin-tar:assemble")
+        result.output.contains("[8.0.1] > Task :distribution:archives:oss-darwin-tar:assemble")
         normalizedOutput(result.output)
                 .contains("distfile /distribution/bwc/bugfix/build/bwc/checkout-8.0/" +
-                        "distribution/archives/darwin-tar/build/install")
+                        "distribution/archives/oss-darwin-tar/build/install")
     }
 
     File setupGitRemote() {

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy

@@ -72,7 +72,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
         def result = gradleRunner("setupDistro", '-g', testProjectDir.newFolder('GUH').path).build()
 
         then:
-        result.task(":distribution:archives:linux-tar:buildExpanded").outcome == TaskOutcome.SUCCESS
+        result.task(":distribution:archives:oss-linux-tar:buildExpanded").outcome == TaskOutcome.SUCCESS
         result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
         assertExtractedDistroIsCreated("build/distro", 'current-marker.txt')
     }
@@ -144,24 +144,24 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
             apply plugin:'base'
             
             // packed distro
-            configurations.create("linux-tar")
+            configurations.create("oss-linux-tar")
             tasks.register("buildBwcTask", Tar) {
                 from('bwc-marker.txt')
                 archiveExtension = "tar.gz"
                 compression = Compression.GZIP
             }
             artifacts {
-                it.add("linux-tar", buildBwcTask)
+                it.add("oss-linux-tar", buildBwcTask)
             }
             
             // expanded distro
-            configurations.create("expanded-linux-tar")
+            configurations.create("expanded-oss-linux-tar")
             def expandedTask = tasks.register("buildBwcExpandedTask", Copy) {
                 from('bwc-marker.txt')
                 into('build/install/elastic-distro')
             }
             artifacts {
-                it.add("expanded-linux-tar", file('build/install')) {
+                it.add("expanded-oss-linux-tar", file('build/install')) {
                     builtBy expandedTask
                     type = 'directory'
                 }
@@ -171,9 +171,9 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
 
     private void localDistroSetup() {
         settingsFile << """
-        include ":distribution:archives:linux-tar"
+        include ":distribution:archives:oss-linux-tar"
         """
-        def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "archives", "linux-tar")
+        def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "archives", "oss-linux-tar")
         new File(bwcSubProjectFolder, 'current-marker.txt') << "current"
         new File(bwcSubProjectFolder, 'build.gradle') << """
             import org.gradle.api.internal.artifacts.ArtifactAttributes;

buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/build.gradle

@@ -23,7 +23,7 @@ subprojects {
     tasks.register('tar', Tar) {
         from('.')
         destinationDirectory.set(file('build/distributions'))
-        archiveBaseName.set("elasticsearch${project.name.startsWith('oss')?'-oss':''}")
+        archiveBaseName.set("elasticsearch-oss")
         archiveVersion.set("8.0.1-SNAPSHOT")
         archiveClassifier.set("darwin-x86_64")
         archiveExtension.set('tar.gz')

buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/settings.gradle

@@ -19,5 +19,4 @@
 
 include ":distribution:bwc:bugfix"
 include ":distribution:bwc:minor"
-include ":distribution:archives:darwin-tar"
 include ":distribution:archives:oss-darwin-tar"

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy

@@ -23,8 +23,6 @@ import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
 import org.gradle.api.Plugin
 import org.gradle.api.Project
-import org.gradle.api.Task
-import org.gradle.api.tasks.TaskProvider
 
 /**
  * Sets up tests for documentation.
@@ -37,7 +35,7 @@ class DocsTestPlugin implements Plugin<Project> {
         project.pluginManager.apply('elasticsearch.standalone-rest-test')
         project.pluginManager.apply('elasticsearch.rest-test')
 
-        String distribution = System.getProperty('tests.distribution', 'default')
+        String distribution = System.getProperty('tests.distribution', 'oss')
         // The distribution can be configured with -Dtests.distribution on the command line
         project.testClusters.integTest.testDistribution = distribution.toUpperCase()
         project.testClusters.integTest.nameCustomization = { it.replace("integTest", "node") }
@@ -51,7 +49,6 @@ class DocsTestPlugin implements Plugin<Project> {
             '\\{version\\}': Version.fromString(VersionProperties.elasticsearch).toString(),
             '\\{version_qualified\\}': VersionProperties.elasticsearch,
             '\\{lucene_version\\}' : VersionProperties.lucene.replaceAll('-snapshot-\\w+$', ''),
-            '\\{build_flavor\\}' : distribution,
             '\\{build_type\\}' : OS.conditionalString().onWindows({"zip"}).onUnix({"tar"}).supply(),
         ]
         project.tasks.register('listSnippets', SnippetsTask) {