build.gradle

/*
 * SPDX-License-Identifier: Apache-2.0
 *
 * The OpenSearch Contributors require contributions made to
 * this file be licensed under the Apache-2.0 license or a
 * compatible open source license.
 *
 * Modifications Copyright OpenSearch Contributors. See
 * GitHub history for details.
 */

/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import org.opensearch.gradle.info.BuildParams

plugins {
  id('com.google.protobuf') version 'latest.release'
  id('opensearch.build')
  id('opensearch.publish')
  id('opensearch.internal-cluster-test')
  id('opensearch.optional-dependencies')
  id('me.champeau.gradle.japicmp') version '0.4.5'
}

publishing {
  publications {
    nebula(MavenPublication) {
      artifactId = 'opensearch'
    }
  }
}

base {
  archivesName = 'opensearch'
}

sourceSets {
  main {
    java {
      srcDir "${buildDir}/generated/source/proto/main/java"
    }
  }
}

dependencies {

  api project(':libs:opensearch-common')
  api project(':libs:opensearch-core')
  api project(":libs:opensearch-compress")
  api project(':libs:opensearch-secure-sm')
  api project(':libs:opensearch-x-content')
  api project(":libs:opensearch-geo")
  api project(":libs:opensearch-telemetry")
  api project(":libs:opensearch-task-commons")

  compileOnly project(':libs:opensearch-plugin-classloader')
  testRuntimeOnly project(':libs:opensearch-plugin-classloader')

  api libs.bundles.lucene

  // utilities
  api project(":libs:opensearch-cli")

  // time handling, remove with java 8 time
  api libs.jodatime

  // percentiles aggregation
  api libs.tdigest
  // percentile ranks aggregation
  api libs.hdrhistogram

  // lucene spatial
  api libs.spatial4j, optional
  api libs.jtscore, optional

  // logging
  api libs.log4japi
  api libs.log4jjul
  api libs.log4jcore, optional
  annotationProcessor libs.log4jcore
  annotationProcessor project(':libs:opensearch-common')

  // jna
  api libs.jna

  // jcraft
  api libs.jzlib

  // reactor
  api libs.reactor.core
  api libs.reactivestreams

  // protobuf
  api libs.protobuf
  api libs.jakartaannotation

  // https://mvnrepository.com/artifact/org.roaringbitmap/RoaringBitmap
  api libs.roaringbitmap
  testImplementation 'org.awaitility:awaitility:4.2.2'
  testImplementation(project(":test:framework")) {
    // tests use the locally compiled version of server
    exclude group: 'org.opensearch', module: 'server'
  }
  internalClusterTestImplementation(project(":test:framework")) {
    exclude group: 'org.opensearch', module: 'server'
  }
}

tasks.withType(JavaCompile).configureEach {
  options.compilerArgs -= '-Xlint:cast'
  options.compilerArgs -= '-Xlint:rawtypes'
  options.compilerArgs -= '-Xlint:unchecked'
}

compileJava {
  options.compilerArgs += ['-processor', ['org.apache.logging.log4j.core.config.plugins.processor.PluginProcessor',
    'org.opensearch.common.annotation.processor.ApiAnnotationProcessor'].join(',')]
}

tasks.named("internalClusterTest").configure {
  // TODO: these run faster with C2 only because they run for so, so long
  jvmArgs -= '-XX:TieredStopAtLevel=1'
}

tasks.named("forbiddenPatterns").configure {
    dependsOn("generateProto")
    exclude '**/*.json'
    exclude '**/*.jmx'
    exclude '**/*.dic'
    exclude '**/*.binary'
    exclude '**/*.st'
    exclude '**/*.meta'
}

tasks.named("testingConventions").configure {
    naming.clear()
    naming {
        Tests {
            baseClass "org.apache.lucene.tests.util.LuceneTestCase"
        }
        IT {
            baseClass "org.opensearch.test.OpenSearchIntegTestCase"
            baseClass "org.opensearch.test.OpenSearchSingleNodeTestCase"
        }
    }
}

// Set to current version by default
def japicmpCompareTarget = System.getProperty("japicmp.compare.version")
if (japicmpCompareTarget == null) { /* use latest released version */
    // Read the list from maven central.
    // Fetch the metadata and parse the xml into Version instances, pick the latest one
    japicmpCompareTarget = new URL('https://repo1.maven.org/maven2/org/opensearch/opensearch/maven-metadata.xml').openStream().withStream { s ->
        new XmlParser().parse(s)
           .versioning.versions.version
           .collect { it.text() }.findAll { it ==~ /\d+\.\d+\.\d+/ }
           .collect { org.opensearch.gradle.Version.fromString(it) }
           .toSorted()
           .last()
           .toString()
        }
}

def generateModulesList = tasks.register("generateModulesList") {
    List<String> modules = project(':modules').subprojects.collect { it.name }
    File modulesFile = new File(buildDir, 'generated-resources/modules.txt')
    processResources.from(modulesFile)
    inputs.property('modules', modules)
    outputs.file(modulesFile)
    doLast {
        modulesFile.parentFile.mkdirs()
        modulesFile.setText(modules.join('\n'), 'UTF-8')
    }
}

def generatePluginsList = tasks.register("generatePluginsList") {
    Set<String> plugins = new TreeSet<>(project(':plugins').childProjects.keySet())
    plugins.remove('example')

    File pluginsFile = new File(buildDir, 'generated-resources/plugins.txt')
    processResources.from(pluginsFile)
    inputs.property('plugins', plugins)
    outputs.file(pluginsFile)
    doLast {
        pluginsFile.parentFile.mkdirs()
        pluginsFile.setText(plugins.join('\n'), 'UTF-8')
    }
}

protobuf {
  protoc {
    artifact = "com.google.protobuf:protoc:${versions.protobuf}"
  }

  generateProtoTasks {
    all().each { task ->
      task.builtins {
        java {
          option "annotate_code"
        }
      }
    }
  }
}

tasks.named("processResources").configure {
    dependsOn generateModulesList, generatePluginsList
}

tasks.named("thirdPartyAudit").configure {
    ignoreMissingClasses(
        *[
            // from com.fasterxml.jackson.dataformat.yaml.YAMLMapper (jackson-dataformat-yaml)
            'com.fasterxml.jackson.databind.ObjectMapper',

            // from log4j
            'com.conversantmedia.util.concurrent.SpinPolicy',
            'com.fasterxml.jackson.annotation.JsonInclude$Include',
            'com.fasterxml.jackson.databind.DeserializationContext',
            'com.fasterxml.jackson.databind.DeserializationFeature',
            'com.fasterxml.jackson.databind.JsonMappingException',
            'com.fasterxml.jackson.databind.JsonNode',
            'com.fasterxml.jackson.databind.Module$SetupContext',
            'com.fasterxml.jackson.databind.ObjectReader',
            'com.fasterxml.jackson.databind.ObjectWriter',
            'com.fasterxml.jackson.databind.SerializerProvider',
            'com.fasterxml.jackson.databind.deser.std.StdDeserializer',
            'com.fasterxml.jackson.databind.deser.std.StdScalarDeserializer',
            'com.fasterxml.jackson.databind.module.SimpleModule',
            'com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter',
            'com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider',
            'com.fasterxml.jackson.databind.ser.std.StdScalarSerializer',
            'com.fasterxml.jackson.databind.ser.std.StdSerializer',
            'com.fasterxml.jackson.dataformat.xml.JacksonXmlModule',
            'com.fasterxml.jackson.dataformat.xml.XmlMapper',
            'com.fasterxml.jackson.dataformat.xml.util.DefaultXmlPrettyPrinter',
            'com.fasterxml.jackson.databind.node.ObjectNode',
            'org.fusesource.jansi.Ansi',
            'org.fusesource.jansi.AnsiRenderer$Code',
            'com.lmax.disruptor.EventFactory',
            'com.lmax.disruptor.EventTranslator',
            'com.lmax.disruptor.EventTranslatorTwoArg',
            'com.lmax.disruptor.EventTranslatorVararg',
            'com.lmax.disruptor.ExceptionHandler',
            'com.lmax.disruptor.LifecycleAware',
            'com.lmax.disruptor.RingBuffer',
            'com.lmax.disruptor.Sequence',
            'com.lmax.disruptor.SequenceReportingEventHandler',
            'com.lmax.disruptor.WaitStrategy',
            'com.lmax.disruptor.dsl.Disruptor',
            'com.lmax.disruptor.dsl.ProducerType',
            'javax.jms.Connection',
            'javax.jms.ConnectionFactory',
            'javax.jms.Destination',
            'javax.jms.JMSException',
            'javax.jms.MapMessage',
            'javax.jms.Message',
            'javax.jms.MessageConsumer',
            'javax.jms.MessageProducer',
            'javax.jms.Session',
            'javax.mail.Authenticator',
            'javax.mail.Message$RecipientType',
            'javax.mail.PasswordAuthentication',
            'javax.mail.Session',
            'javax.mail.Transport',
            'javax.mail.internet.InternetAddress',
            'javax.mail.internet.InternetHeaders',
            'javax.mail.internet.MimeMessage',
            'javax.mail.internet.MimeMultipart',
            'javax.mail.internet.MimeUtility',
            'org.apache.commons.compress.compressors.CompressorStreamFactory',
            'org.apache.commons.compress.utils.IOUtils',
            'org.apache.commons.csv.CSVFormat',
            'org.apache.commons.csv.QuoteMode',
            'org.apache.kafka.clients.producer.Producer',
            'org.apache.kafka.clients.producer.RecordMetadata',
            'org.codehaus.stax2.XMLStreamWriter2',
            'org.jctools.queues.MpscArrayQueue',
            'org.osgi.framework.Bundle',
            'org.osgi.framework.BundleActivator',
            'org.osgi.framework.BundleContext',
            'org.osgi.framework.BundleEvent',
            'org.osgi.framework.BundleReference',
            'org.osgi.framework.FrameworkUtil',
            'org.osgi.framework.ServiceRegistration',
            'org.osgi.framework.SynchronousBundleListener',
            'org.osgi.framework.wiring.BundleWire',
            'org.osgi.framework.wiring.BundleWiring',
            'org.zeromq.ZMQ$Context',
            'org.zeromq.ZMQ$Socket',
            'org.zeromq.ZMQ',

            // from org.locationtech.spatial4j.io.GeoJSONReader (spatial4j)
            'org.noggit.JSONParser',

            // from lucene-spatial
            'com.fasterxml.jackson.databind.JsonSerializer',
            'com.fasterxml.jackson.databind.JsonDeserializer',
            'com.fasterxml.jackson.databind.node.ArrayNode',
            'com.google.common.geometry.S2Cell',
            'com.google.common.geometry.S2CellId',
            'com.google.common.geometry.S2Projections',
            'com.google.common.geometry.S2Point',
            'com.google.common.geometry.S2$Metric',
            'com.google.common.geometry.S2LatLng'
        ] + (BuildParams.runtimeJavaVersion < JavaVersion.VERSION_20) ? [] : [
            'jdk.incubator.vector.ByteVector',
            'jdk.incubator.vector.FloatVector',
            'jdk.incubator.vector.IntVector',
            'jdk.incubator.vector.ShortVector',
            'jdk.incubator.vector.Vector',
            'jdk.incubator.vector.VectorOperators',
            'jdk.incubator.vector.VectorSpecies'
        ]
    )
    ignoreViolations(
            'com.google.protobuf.MessageSchema',
            'com.google.protobuf.UnsafeUtil',
            'com.google.protobuf.UnsafeUtil$1',
            'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
            'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
            'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
            'com.google.protobuf.UnsafeUtil$MemoryAccessor',
            'org.apache.logging.log4j.core.util.internal.UnsafeUtil',
            'org.apache.logging.log4j.core.util.internal.UnsafeUtil$1',
            'reactor.core.publisher.Traces$SharedSecretsCallSiteSupplierFactory$TracingException'
    )
}

tasks.named("dependencyLicenses").configure {
    mapping from: /jackson-.*/, to: 'jackson'
    mapping from: /reactor-.*/, to: 'reactor'
    mapping from: /lucene-.*/, to: 'lucene'
    dependencies = project.configurations.runtimeClasspath.incoming.artifactView {
        componentFilter {
            it instanceof ModuleComponentIdentifier &&
                (it.group.startsWith('org.opensearch') == false ||
                    // keep the following org.opensearch jars in
                    (it.name == 'jna' ||
                            it.name == 'securesm'))
        }
    }.files
}

tasks.named("filepermissions").configure {
  mustRunAfter("generateProto")
}

tasks.named("licenseHeaders").configure {
    dependsOn("generateProto")
    // Ignore our vendored version of Google Guice
    excludes << 'org/opensearch/common/inject/**/*'
    // Ignore temporary copies of impending 8.7 Lucene classes
    excludes << 'org/apache/lucene/search/RegExp87*'
    excludes << 'org/apache/lucene/search/RegexpQuery87*'
    // Ignore for protobuf generated code
    excludes << 'org/opensearch/extensions/proto/*'
}

tasks.test {
    environment "node.roles.test", "[]"
    if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
        jvmArgs += ["--add-opens", "java.base/java.nio.file=ALL-UNNAMED"]
    }
}

tasks.named("sourcesJar").configure {
  // Ignore duplicates for protobuf generated code (main and generatedSources).
  filesMatching("**/proto/*") {
    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
  }
}

/** Compares the current build against a laltest released version or the version supplied through 'japicmp.compare.version' system property */
tasks.register("japicmp", me.champeau.gradle.japicmp.JapicmpTask) {
    logger.info("Comparing public APIs from ${version} to ${japicmpCompareTarget}")
    // See please https://github.com/siom79/japicmp/issues/201
    compatibilityChangeExcludes = [ "METHOD_ABSTRACT_NOW_DEFAULT" ]
    oldClasspath.from(files("${buildDir}/japicmp-target/opensearch-${japicmpCompareTarget}.jar"))
    newClasspath.from(tasks.named('jar'))
    onlyModified = true
    failOnModification = true
    ignoreMissingClasses = true
    failOnSourceIncompatibility = true
    annotationIncludes = ['@org.opensearch.common.annotation.PublicApi', '@org.opensearch.common.annotation.DeprecatedApi']
    annotationExcludes = ['@org.opensearch.common.annotation.InternalApi']
    txtOutputFile = layout.buildDirectory.file("reports/java-compatibility/report.txt")
    htmlOutputFile = layout.buildDirectory.file("reports/java-compatibility/report.html")
    dependsOn downloadJapicmpCompareTarget
}

/** If the Java API Comparison task failed, print a hint if the change should be merged from its target branch */
gradle.taskGraph.afterTask { Task task, TaskState state ->
    if (task.name == 'japicmp' && state.failure != null) {
        logger.info("Public APIs changes incompatiable with ${japicmpCompareTarget} target have been detected")
    }
}

/** Downloads latest released version from maven repository */
tasks.register("downloadJapicmpCompareTarget", Copy) {
    def mavenSnapshotRepoUrl = "https://aws.oss.sonatype.org/content/repositories/snapshots/"
    def groupId = "org.opensearch"
    def artifactId = "opensearch"

    // Add repository for snapshot artifacts if japicmp compare target version is snapshot
    if (japicmpCompareTarget.endsWith("-SNAPSHOT")) {
        def repos = project.getRepositories();
        MavenArtifactRepository opensearchRepo = repos.maven(repo -> {
            repo.setName("opensearch-snapshots");
            repo.setUrl(mavenSnapshotRepoUrl);
        });

        repos.exclusiveContent(exclusiveRepo -> {
            exclusiveRepo.filter(descriptor -> descriptor.includeGroup(groupId));
            exclusiveRepo.forRepositories(opensearchRepo);
        });
    }

    configurations {
        japicmpCompareTargetArtifact {
            exclude group: 'org.apache.lucene'
        }
    }

    dependencies {
        japicmpCompareTargetArtifact("${groupId}:${artifactId}:${japicmpCompareTarget}:")
    }

    from configurations.japicmpCompareTargetArtifact
    into "$buildDir/japicmp-target"
}

/** Check if the sha is in the current history */
def inHistory(String sha) {
    try {
      def commandCheckSha = "git merge-base --is-ancestor ${sha} HEAD"
      commandCheckSha.execute()
      return true
    } catch (Exception) {
      return false
    }
}

/** Extracts the Git SHA used to build a jar from its manifest */
def getGitShaFromJar(String jarPath) {
    def sha = ''
    try {
        // Open the JAR file
        def jarFile = new java.util.jar.JarFile(jarPath)
        // Get the manifest from the JAR file
        def manifest = jarFile.manifest
        def attributes = manifest.mainAttributes
        // Assuming the Git SHA is stored under an attribute named 'Git-SHA'
        sha = attributes.getValue('Change')
        jarFile.close()
    } catch (IOException e) {
        println "Failed to read the JAR file: $e.message"
    }
    return sha
}