Skip to content

Commit

Permalink
Separates integration test dependencies and updates build.gradle files
Browse files Browse the repository at this point in the history 
Signed-off-by: Darshit Chanpura <dchanp@amazon.com>
  • Loading branch information
@DarshitChanpura
DarshitChanpura committed Jan 15, 2025
1 parent e1e876f commit 45e0960
Show file tree
Hide file tree
Showing 3 changed files with 94 additions and 157 deletions.
78 changes: 53 additions & 25 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -499,13 +499,66 @@ configurations {
force "org.checkerframework:checker-qual:3.48.4"
force "ch.qos.logback:logback-classic:1.5.16"
force "commons-io:commons-io:2.18.0"
force "com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.2"
force "org.hamcrest:hamcrest:2.2"
force "org.mockito:mockito-core:5.15.2"
force "net.bytebuddy:byte-buddy:1.15.11"
}
}

integrationTestImplementation.extendsFrom implementation
integrationTestRuntimeOnly.extendsFrom runtimeOnly
}

allprojects {
configurations {
integrationTestImplementation.extendsFrom implementation
}
dependencies {
//integration test framework:
integrationTestImplementation('com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.2') {
exclude(group: 'junit', module: 'junit')
}
integrationTestImplementation 'junit:junit:4.13.2'
integrationTestImplementation("org.opensearch.plugin:reindex-client:${opensearch_version}"){
exclude(group: 'org.slf4j', module: 'slf4j-api')
}
integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}"
integrationTestImplementation 'commons-io:commons-io:2.18.0'
integrationTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"
integrationTestImplementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}"
integrationTestImplementation 'org.hamcrest:hamcrest:2.2'
integrationTestImplementation "org.bouncycastle:bcpkix-jdk18on:${versions.bouncycastle}"
integrationTestImplementation "org.bouncycastle:bcutil-jdk18on:${versions.bouncycastle}"
integrationTestImplementation('org.awaitility:awaitility:4.2.2') {
exclude(group: 'org.hamcrest', module: 'hamcrest')
}
integrationTestImplementation 'com.unboundid:unboundid-ldapsdk:4.0.14'
integrationTestImplementation "org.opensearch.plugin:mapper-size:${opensearch_version}"
integrationTestImplementation "org.apache.httpcomponents:httpclient-cache:4.5.14"
integrationTestImplementation "org.apache.httpcomponents:httpclient:4.5.14"
integrationTestImplementation "org.apache.httpcomponents:fluent-hc:4.5.14"
integrationTestImplementation "org.apache.httpcomponents:httpcore:4.4.16"
integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5"
integrationTestImplementation "org.mockito:mockito-core:5.15.2"
integrationTestImplementation "org.passay:passay:1.6.6"
integrationTestImplementation "org.opensearch:opensearch:${opensearch_version}"
integrationTestImplementation "org.opensearch.plugin:transport-netty4-client:${opensearch_version}"
integrationTestImplementation "org.opensearch.plugin:aggs-matrix-stats-client:${opensearch_version}"
integrationTestImplementation "org.opensearch.plugin:parent-join-client:${opensearch_version}"
integrationTestImplementation 'com.password4j:password4j:1.8.2'
integrationTestImplementation "com.google.guava:guava:${guava_version}"
integrationTestImplementation "org.apache.commons:commons-lang3:${versions.commonslang}"
integrationTestImplementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
integrationTestImplementation 'org.greenrobot:eventbus-java:3.3.1'
integrationTestImplementation('com.flipkart.zjsonpatch:zjsonpatch:0.4.16'){
exclude(group:'com.fasterxml.jackson.core')
}
integrationTestImplementation 'org.slf4j:slf4j-api:2.0.12'
integrationTestImplementation 'com.selectivem.collections:special-collections-complete:1.4.0'
}
}

//create source set 'integrationTest'
//add classes from the main source set to the compilation and runtime classpaths of the integrationTest
sourceSets {
Expand Down Expand Up @@ -724,31 +777,6 @@ dependencies {

compileOnly "org.opensearch:opensearch:${opensearch_version}"

//integration test framework:
integrationTestImplementation('com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.2') {
exclude(group: 'junit', module: 'junit')
}
integrationTestImplementation 'junit:junit:4.13.2'
integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}"
integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}"
integrationTestImplementation 'commons-io:commons-io:2.18.0'
integrationTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"
integrationTestImplementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}"
integrationTestImplementation 'org.hamcrest:hamcrest:2.2'
integrationTestImplementation "org.bouncycastle:bcpkix-jdk18on:${versions.bouncycastle}"
integrationTestImplementation "org.bouncycastle:bcutil-jdk18on:${versions.bouncycastle}"
integrationTestImplementation('org.awaitility:awaitility:4.2.2') {
exclude(group: 'org.hamcrest', module: 'hamcrest')
}
integrationTestImplementation 'com.unboundid:unboundid-ldapsdk:4.0.14'
integrationTestImplementation "org.opensearch.plugin:mapper-size:${opensearch_version}"
integrationTestImplementation "org.apache.httpcomponents:httpclient-cache:4.5.14"
integrationTestImplementation "org.apache.httpcomponents:httpclient:4.5.14"
integrationTestImplementation "org.apache.httpcomponents:fluent-hc:4.5.14"
integrationTestImplementation "org.apache.httpcomponents:httpcore:4.4.16"
integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5"
integrationTestImplementation "org.mockito:mockito-core:5.15.2"

//spotless
implementation('com.google.googlejavaformat:google-java-format:1.25.2') {
exclude group: 'com.google.guava'
Expand Down
172 changes: 41 additions & 131 deletions sample-resource-plugin/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,6 @@

apply plugin: 'opensearch.opensearchplugin'
apply plugin: 'opensearch.testclusters'
apply plugin: 'opensearch.java-rest-test'

import org.opensearch.gradle.test.RestIntegTestTask

java {
sourceCompatibility = JavaVersion.VERSION_21
Expand All @@ -20,8 +17,13 @@ opensearchplugin {
classname 'org.opensearch.sample.SampleResourcePlugin'
}

dependencyLicenses.enabled = false
thirdPartyAudit.enabled = false
loggerUsageCheck.enabled = false
tasks.test.enabled=false
validateNebulaPom.enabled=false

ext {
projectSubstitutions = [:]
licenseFile = rootProject.file('LICENSE.txt')
noticeFile = rootProject.file('NOTICE.txt')
opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
Expand All @@ -31,7 +33,6 @@ ext {
version_tokens = opensearch_version.tokenize('-')
opensearch_build = version_tokens[0] + '.0'


if (buildVersionQualifier) {
opensearch_build += "-${buildVersionQualifier}"
}
Expand All @@ -46,144 +47,53 @@ repositories {
maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
}

configurations.all {
resolutionStrategy {
force 'com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.2',
'org.hamcrest:hamcrest:2.2',
'org.apache.httpcomponents:httpclient:4.5.14',
'org.apache.httpcomponents:httpcore:4.4.16',
'org.mockito:mockito-core:5.15.2',
'net.bytebuddy:byte-buddy:1.15.11',
'commons-codec:commons-codec:1.16.1',
'com.fasterxml.jackson.core:jackson-databind:2.18.2',
'com.fasterxml.jackson.core:jackson-databind:2.18.2'
}
}

dependencies {
// Main implementation dependencies
implementation "org.opensearch:opensearch-resource-sharing-spi:${opensearch_build}"
implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
}

dependencyLicenses.enabled = false
thirdPartyAudit.enabled = false

def es_tmp_dir = rootProject.file('build/private/es_tmp').absoluteFile
es_tmp_dir.mkdirs()

File repo = file("$buildDir/testclusters/repo")
def _numNodes = findProperty('numNodes') as Integer ?: 1

licenseHeaders.enabled = true
validateNebulaPom.enabled = false
testingConventions.enabled = false
loggerUsageCheck.enabled = false

javaRestTest.dependsOn(rootProject.assemble)
javaRestTest {
systemProperty 'tests.security.manager', 'false'
// Integration test dependencies
integrationTestImplementation rootProject.sourceSets.integrationTest.output
integrationTestImplementation rootProject.sourceSets.main.output
}
testClusters.javaRestTest {
testDistribution = 'INTEG_TEST'
}

task integTest(type: RestIntegTestTask) {
description = "Run tests against a cluster"
testClassesDirs = sourceSets.test.output.classesDirs
classpath = sourceSets.test.runtimeClasspath
}
tasks.named("check").configure { dependsOn(integTest) }

integTest {
if (project.hasProperty('excludeTests')) {
project.properties['excludeTests']?.replaceAll('\\s', '')?.split('[,;]')?.each {
exclude "${it}"
}
}
systemProperty 'tests.security.manager', 'false'
systemProperty 'java.io.tmpdir', es_tmp_dir.absolutePath

systemProperty "https", System.getProperty("https")
systemProperty "user", System.getProperty("user")
systemProperty "password", System.getProperty("password")
// Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for
// requests. The 'doFirst' delays reading the debug setting on the cluster till execution time.
doFirst {
// Tell the test JVM if the cluster JVM is running under a debugger so that tests can
// use longer timeouts for requests.
def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null
systemProperty 'cluster.debug', isDebuggingCluster
// Set number of nodes system property to be used in tests
systemProperty 'cluster.number_of_nodes', "${_numNodes}"
// There seems to be an issue when running multi node run or integ tasks with unicast_hosts
// not being written, the waitForAllConditions ensures it's written
getClusters().forEach { cluster ->
cluster.waitForAllConditions()
}
}

// The -Dcluster.debug option makes the cluster debuggable; this makes the tests debuggable
if (System.getProperty("test.debug") != null) {
jvmArgs '-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=8000'
}
if (System.getProperty("tests.rest.bwcsuite") == null) {
filter {
excludeTestsMatching "org.opensearch.security.sampleextension.bwc.*IT"
sourceSets {
integrationTest {
java {
srcDir file('src/integrationTest/java')
compileClasspath += sourceSets.main.output
runtimeClasspath += sourceSets.main.output
}
}
}
project.getTasks().getByName('bundlePlugin').dependsOn(rootProject.tasks.getByName('build'))
Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin");
Zip rootBundle = (Zip) rootProject.getTasks().getByName("bundlePlugin");
integTest.dependsOn(bundle)
integTest.getClusters().forEach{c -> {
c.plugin(rootProject.getObjects().fileProperty().value(rootBundle.getArchiveFile()))
c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile()))
}}

testClusters.integTest {
testDistribution = 'INTEG_TEST'

// Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1
if (_numNodes > 1) numberOfNodes = _numNodes
// When running integration tests it doesn't forward the --debug-jvm to the cluster anymore
// i.e. we have to use a custom property to flag when we want to debug OpenSearch JVM
// since we also support multi node integration tests we increase debugPort per node
if (System.getProperty("cluster.debug") != null) {
def debugPort = 5005
nodes.forEach { node ->
node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}")
debugPort += 1
resources {
srcDir file('src/integrationTest/resources')
}
}
setting 'path.repo', repo.absolutePath
}

afterEvaluate {
testClusters.integTest.nodes.each { node ->
def plugins = node.plugins
def firstPlugin = plugins.get(0)
if (firstPlugin.provider == project.bundlePlugin.archiveFile) {
plugins.remove(0)
plugins.add(firstPlugin)
}
tasks.register("integrationTest", Test) {
description = 'Run integration tests for the subproject.'
group = 'verification'

testClassesDirs = sourceSets.integrationTest.output.classesDirs
classpath = sourceSets.integrationTest.runtimeClasspath

node.extraConfigFile("kirk.pem", file("src/test/resources/security/kirk.pem"))
node.extraConfigFile("kirk-key.pem", file("src/test/resources/security/kirk-key.pem"))
node.extraConfigFile("esnode.pem", file("src/test/resources/security/esnode.pem"))
node.extraConfigFile("esnode-key.pem", file("src/test/resources/security/esnode-key.pem"))
node.extraConfigFile("root-ca.pem", file("src/test/resources/security/root-ca.pem"))
node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
node.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
node.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
node.setting("plugins.security.ssl.http.enabled", "true")
node.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
node.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
node.setting("plugins.security.allow_unsafe_democertificates", "true")
node.setting("plugins.security.allow_default_init_securityindex", "true")
node.setting("plugins.security.authcz.admin_dn", "\n - CN=kirk,OU=client,O=client,L=test,C=de")
node.setting("plugins.security.audit.type", "internal_opensearch")
node.setting("plugins.security.enable_snapshot_restore_privilege", "true")
node.setting("plugins.security.check_snapshot_restore_write_privileges", "true")
node.setting("plugins.security.restapi.roles_enabled", "[\"all_access\", \"security_rest_api_access\"]")
}
}

run {
doFirst {
// There seems to be an issue when running multi node run or integ tasks with unicast_hosts
// not being written, the waitForAllConditions ensures it's written
getClusters().forEach { cluster ->
cluster.waitForAllConditions()
}
}
useCluster testClusters.integTest
// Ensure integrationTest task depends on the root project's compile task
tasks.named("integrationTest").configure {
dependsOn rootProject.tasks.named("compileIntegrationTestJava")
}
1 change: 0 additions & 1 deletion spi/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ repositories {

dependencies {
compileOnly "org.opensearch:opensearch:${opensearch_version}"
testImplementation "org.opensearch.test:framework:${opensearch_version}"
}

java {
Expand Down

0 comments on commit 45e0960

Please sign in to comment.