Skip to content

Commit

Permalink
Proto generation spreading (google#174)
Browse files Browse the repository at this point in the history 
Spread proto generation over multiple protoc calls on systems that limit command line length, e.g., Windows. Once the command length exceeds the OS limit, it calls with the current arguments and resets the builder.

This is related to google#167, allowing this issue to be avoided when many proto files are present, until protocolbuffers/protobuf#274 is completed.
  • Loading branch information
@Gegy @zhangkun83
Gegy authored and zhangkun83 committed Nov 7, 2018
1 parent 4138180 commit 1e602b1
Show file tree
Hide file tree
Showing 2 changed files with 151 additions and 10 deletions.
74 changes: 64 additions & 10 deletions src/main/groovy/com/google/protobuf/gradle/GenerateProtoTask.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ package com.google.protobuf.gradle

import com.google.common.base.Preconditions
import com.google.common.collect.ImmutableList
import com.google.common.primitives.Ints

import org.gradle.api.DefaultTask
import org.gradle.api.GradleException
Expand All @@ -46,6 +47,8 @@ import org.gradle.util.ConfigureUtil
*/
// TODO(zhangkun83): add per-plugin output dir reconfiguraiton.
public class GenerateProtoTask extends DefaultTask {
static final int VISTA_CMD_LENGTH_LIMIT = 8191
static final int XP_CMD_LENGTH_LIMIT = 2047

private final List includeDirs = []
private final NamedDomainObjectContainer<PluginOptions> builtins
Expand Down Expand Up @@ -372,13 +375,13 @@ public class GenerateProtoTask extends DefaultTask {
List<String> dirs = includeDirs*.path.collect { "-I${it}" }
logger.debug "ProtobufCompile using directories ${dirs}"
logger.debug "ProtobufCompile using files ${protoFiles}"
List<String> cmd = [ tools.protoc.path ]
cmd.addAll(dirs)
List<String> baseCmd = [ tools.protoc.path ]
baseCmd.addAll(dirs)

// Handle code generation built-ins
builtins.each { builtin ->
String outPrefix = makeOptionsPrefix(builtin.options)
cmd += "--${builtin.name}_out=${outPrefix}${getOutputDir(builtin)}"
baseCmd += "--${builtin.name}_out=${outPrefix}${getOutputDir(builtin)}"
}

// Handle code generation plugins
Expand All @@ -389,8 +392,8 @@ public class GenerateProtoTask extends DefaultTask {
throw new GradleException("Codegen plugin ${name} not defined")
}
String pluginOutPrefix = makeOptionsPrefix(plugin.options)
cmd += "--${name}_out=${pluginOutPrefix}${getOutputDir(plugin)}"
cmd += "--plugin=protoc-gen-${name}=${locator.path}"
baseCmd += "--${name}_out=${pluginOutPrefix}${getOutputDir(plugin)}"
baseCmd += "--plugin=protoc-gen-${name}=${locator.path}"
}

if (generateDescriptorSet) {
Expand All @@ -401,17 +404,24 @@ public class GenerateProtoTask extends DefaultTask {
if (!folder.exists()) {
folder.mkdirs()
}
cmd += "--descriptor_set_out=${path}"
baseCmd += "--descriptor_set_out=${path}"
if (descriptorSetOptions.includeImports) {
cmd += "--include_imports"
baseCmd += "--include_imports"
}
if (descriptorSetOptions.includeSourceInfo) {
cmd += "--include_source_info"
baseCmd += "--include_source_info"
}
}

cmd.addAll protoFiles
logger.log(LogLevel.INFO, cmd.toString())
List<String> cmds = generateCmds(baseCmd.join(" "), protoFiles, getCmdLengthLimit())
for (String cmd : cmds) {
compileFiles(cmd)
}
}

private void compileFiles(String cmd) {
logger.log(LogLevel.INFO, cmd)

StringBuffer stdout = new StringBuffer()
StringBuffer stderr = new StringBuffer()
Process result = cmd.execute()
Expand All @@ -424,4 +434,48 @@ public class GenerateProtoTask extends DefaultTask {
}
}

static List<String> generateCmds(String baseCmd, List<File> protoFiles, int cmdLengthLimit) {
List<String> cmds = []
if (!protoFiles.isEmpty()) {
StringBuilder currentCmd = new StringBuilder(baseCmd)
for (File proto: protoFiles) {
String protoFileName = proto
// Check if appending the next proto string will overflow the cmd length limit
if (currentCmd.length() + protoFileName.length() + 1 > cmdLengthLimit) {
// Add the current cmd before overflow
cmds.add(currentCmd.toString())
currentCmd.setLength(baseCmd.length())
}
// Append the proto file to the command
currentCmd.append(" ").append(protoFileName)
}
// Add the last cmd for execution
cmds.add(currentCmd.toString())
}
return cmds
}

static int getCmdLengthLimit() {
return getCmdLengthLimit(System.getProperty("os.name"), System.getProperty("os.version"))
}

static int getCmdLengthLimit(String os, String version) {
// Check if operating system is Windows
if (os != null && os.toLowerCase(Locale.ROOT).indexOf("win") > -1) {
if (version != null) {
int idx = version.indexOf('.')
if (idx > 0) {
// If the major version is > 5, we are on Windows Vista or higher
int majorVersion = Ints.tryParse(version[0..(idx - 1)]) ?: 0
if (majorVersion > 5) {
return VISTA_CMD_LENGTH_LIMIT
}
}
}
// Failed to read version, assume Windows XP or lower
return XP_CMD_LENGTH_LIMIT
}
return Integer.MAX_VALUE
}

}
87 changes: 87 additions & 0 deletions src/test/groovy/com/google/plugins/ProtobufJavaPluginTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,4 +153,91 @@ class ProtobufJavaPluginTest extends Specification {
where:
gradleVersion << GRADLE_VERSIONS
}

void "test generateCmds should split commands when limit exceeded"() {
given: "a cmd length limit and two proto files"

String baseCmd = "protoc"
List<File> protoFiles = [ new File("short.proto"), new File("long_proto_name.proto") ]
int cmdLengthLimit = 32

when: "the commands are generated"

List<String> cmds = GenerateProtoTask.generateCmds(baseCmd, protoFiles, cmdLengthLimit)

then: "it splits appropriately"
cmds.size() == 2 && cmds[0] == "protoc short.proto" && cmds[1] == "protoc long_proto_name.proto"
}

void "test generateCmds should not split commands when under limit"() {
given: "a cmd length limit and two proto files"

String baseCmd = "protoc"
List<File> protoFiles = [ new File("short.proto"), new File("long_proto_name.proto") ]
int cmdLengthLimit = 64

when: "the commands are generated"

List<String> cmds = GenerateProtoTask.generateCmds(baseCmd, protoFiles, cmdLengthLimit)

then: "it splits appropriately"
cmds.size() == 1 && cmds[0] == "protoc short.proto long_proto_name.proto"
}

void "test generateCmds should not return commands when no protos are given"() {
given: "a cmd length limit and no proto files"

String baseCmd = "protoc"
List<File> protoFiles = []
int cmdLengthLimit = 32

when: "the commands are generated"

List<String> cmds = GenerateProtoTask.generateCmds(baseCmd, protoFiles, cmdLengthLimit)

then: "it returns no commands"
cmds.isEmpty()
}

void "test getCmdLengthLimit returns correct limit for Windows XP"() {
given: "Windows OS at major version 5"

String os = "windows"
String version = "5.0.0"

when: "the command length limit is queried"

int limit = GenerateProtoTask.getCmdLengthLimit(os, version)

then: "it returns the XP limit"
limit == GenerateProtoTask.XP_CMD_LENGTH_LIMIT
}

void "test getCmdLengthLimit returns correct limit for Windows Vista"() {
given: "Windows OS at major version 6"

String os = "Windows"
String version = "6.0.0"

when: "the command length limit is queried"

int limit = GenerateProtoTask.getCmdLengthLimit(os, version)

then: "it returns the Vista limit"
limit == GenerateProtoTask.VISTA_CMD_LENGTH_LIMIT
}

void "test getCmdLengthLimit returns correct limit for non-Windows OS"() {
given: "MacOS X at major version 10"

String os = "Mac OS X"
String version = "10.0.0"

when: "the command length limit is queried"

int limit = GenerateProtoTask.getCmdLengthLimit(os, version)

then: "it returns maximum integer value"
limit == Integer.MAX_VALUE
}
}

0 comments on commit 1e602b1

Please sign in to comment.