From 1e602b167cabc2caf201fac23a9c889784a231bb Mon Sep 17 00:00:00 2001 From: gegy1000 Date: Wed, 22 Nov 2017 19:43:05 +0200 Subject: [PATCH] Proto generation spreading (#174) Spread proto generation over multiple protoc calls on systems that limit command line length, e.g., Windows. Once the command length exceeds the OS limit, it calls with the current arguments and resets the builder. This is related to #167, allowing this issue to be avoided when many proto files are present, until google/protobuf#274 is completed. --- .../protobuf/gradle/GenerateProtoTask.groovy | 74 +++++++++++++--- .../plugins/ProtobufJavaPluginTest.groovy | 87 +++++++++++++++++++ 2 files changed, 151 insertions(+), 10 deletions(-) diff --git a/src/main/groovy/com/google/protobuf/gradle/GenerateProtoTask.groovy b/src/main/groovy/com/google/protobuf/gradle/GenerateProtoTask.groovy index 327bba23..a40af3fa 100644 --- a/src/main/groovy/com/google/protobuf/gradle/GenerateProtoTask.groovy +++ b/src/main/groovy/com/google/protobuf/gradle/GenerateProtoTask.groovy @@ -31,6 +31,7 @@ package com.google.protobuf.gradle import com.google.common.base.Preconditions import com.google.common.collect.ImmutableList +import com.google.common.primitives.Ints import org.gradle.api.DefaultTask import org.gradle.api.GradleException @@ -46,6 +47,8 @@ import org.gradle.util.ConfigureUtil */ // TODO(zhangkun83): add per-plugin output dir reconfiguraiton. public class GenerateProtoTask extends DefaultTask { + static final int VISTA_CMD_LENGTH_LIMIT = 8191 + static final int XP_CMD_LENGTH_LIMIT = 2047 private final List includeDirs = [] private final NamedDomainObjectContainer builtins @@ -372,13 +375,13 @@ public class GenerateProtoTask extends DefaultTask { List dirs = includeDirs*.path.collect { "-I${it}" } logger.debug "ProtobufCompile using directories ${dirs}" logger.debug "ProtobufCompile using files ${protoFiles}" - List cmd = [ tools.protoc.path ] - cmd.addAll(dirs) + List baseCmd = [ tools.protoc.path ] + baseCmd.addAll(dirs) // Handle code generation built-ins builtins.each { builtin -> String outPrefix = makeOptionsPrefix(builtin.options) - cmd += "--${builtin.name}_out=${outPrefix}${getOutputDir(builtin)}" + baseCmd += "--${builtin.name}_out=${outPrefix}${getOutputDir(builtin)}" } // Handle code generation plugins @@ -389,8 +392,8 @@ public class GenerateProtoTask extends DefaultTask { throw new GradleException("Codegen plugin ${name} not defined") } String pluginOutPrefix = makeOptionsPrefix(plugin.options) - cmd += "--${name}_out=${pluginOutPrefix}${getOutputDir(plugin)}" - cmd += "--plugin=protoc-gen-${name}=${locator.path}" + baseCmd += "--${name}_out=${pluginOutPrefix}${getOutputDir(plugin)}" + baseCmd += "--plugin=protoc-gen-${name}=${locator.path}" } if (generateDescriptorSet) { @@ -401,17 +404,24 @@ public class GenerateProtoTask extends DefaultTask { if (!folder.exists()) { folder.mkdirs() } - cmd += "--descriptor_set_out=${path}" + baseCmd += "--descriptor_set_out=${path}" if (descriptorSetOptions.includeImports) { - cmd += "--include_imports" + baseCmd += "--include_imports" } if (descriptorSetOptions.includeSourceInfo) { - cmd += "--include_source_info" + baseCmd += "--include_source_info" } } - cmd.addAll protoFiles - logger.log(LogLevel.INFO, cmd.toString()) + List cmds = generateCmds(baseCmd.join(" "), protoFiles, getCmdLengthLimit()) + for (String cmd : cmds) { + compileFiles(cmd) + } + } + + private void compileFiles(String cmd) { + logger.log(LogLevel.INFO, cmd) + StringBuffer stdout = new StringBuffer() StringBuffer stderr = new StringBuffer() Process result = cmd.execute() @@ -424,4 +434,48 @@ public class GenerateProtoTask extends DefaultTask { } } + static List generateCmds(String baseCmd, List protoFiles, int cmdLengthLimit) { + List cmds = [] + if (!protoFiles.isEmpty()) { + StringBuilder currentCmd = new StringBuilder(baseCmd) + for (File proto: protoFiles) { + String protoFileName = proto + // Check if appending the next proto string will overflow the cmd length limit + if (currentCmd.length() + protoFileName.length() + 1 > cmdLengthLimit) { + // Add the current cmd before overflow + cmds.add(currentCmd.toString()) + currentCmd.setLength(baseCmd.length()) + } + // Append the proto file to the command + currentCmd.append(" ").append(protoFileName) + } + // Add the last cmd for execution + cmds.add(currentCmd.toString()) + } + return cmds + } + + static int getCmdLengthLimit() { + return getCmdLengthLimit(System.getProperty("os.name"), System.getProperty("os.version")) + } + + static int getCmdLengthLimit(String os, String version) { + // Check if operating system is Windows + if (os != null && os.toLowerCase(Locale.ROOT).indexOf("win") > -1) { + if (version != null) { + int idx = version.indexOf('.') + if (idx > 0) { + // If the major version is > 5, we are on Windows Vista or higher + int majorVersion = Ints.tryParse(version[0..(idx - 1)]) ?: 0 + if (majorVersion > 5) { + return VISTA_CMD_LENGTH_LIMIT + } + } + } + // Failed to read version, assume Windows XP or lower + return XP_CMD_LENGTH_LIMIT + } + return Integer.MAX_VALUE + } + } diff --git a/src/test/groovy/com/google/plugins/ProtobufJavaPluginTest.groovy b/src/test/groovy/com/google/plugins/ProtobufJavaPluginTest.groovy index dc9fc499..b43bf14a 100644 --- a/src/test/groovy/com/google/plugins/ProtobufJavaPluginTest.groovy +++ b/src/test/groovy/com/google/plugins/ProtobufJavaPluginTest.groovy @@ -153,4 +153,91 @@ class ProtobufJavaPluginTest extends Specification { where: gradleVersion << GRADLE_VERSIONS } + + void "test generateCmds should split commands when limit exceeded"() { + given: "a cmd length limit and two proto files" + + String baseCmd = "protoc" + List protoFiles = [ new File("short.proto"), new File("long_proto_name.proto") ] + int cmdLengthLimit = 32 + + when: "the commands are generated" + + List cmds = GenerateProtoTask.generateCmds(baseCmd, protoFiles, cmdLengthLimit) + + then: "it splits appropriately" + cmds.size() == 2 && cmds[0] == "protoc short.proto" && cmds[1] == "protoc long_proto_name.proto" + } + + void "test generateCmds should not split commands when under limit"() { + given: "a cmd length limit and two proto files" + + String baseCmd = "protoc" + List protoFiles = [ new File("short.proto"), new File("long_proto_name.proto") ] + int cmdLengthLimit = 64 + + when: "the commands are generated" + + List cmds = GenerateProtoTask.generateCmds(baseCmd, protoFiles, cmdLengthLimit) + + then: "it splits appropriately" + cmds.size() == 1 && cmds[0] == "protoc short.proto long_proto_name.proto" + } + + void "test generateCmds should not return commands when no protos are given"() { + given: "a cmd length limit and no proto files" + + String baseCmd = "protoc" + List protoFiles = [] + int cmdLengthLimit = 32 + + when: "the commands are generated" + + List cmds = GenerateProtoTask.generateCmds(baseCmd, protoFiles, cmdLengthLimit) + + then: "it returns no commands" + cmds.isEmpty() + } + + void "test getCmdLengthLimit returns correct limit for Windows XP"() { + given: "Windows OS at major version 5" + + String os = "windows" + String version = "5.0.0" + + when: "the command length limit is queried" + + int limit = GenerateProtoTask.getCmdLengthLimit(os, version) + + then: "it returns the XP limit" + limit == GenerateProtoTask.XP_CMD_LENGTH_LIMIT + } + + void "test getCmdLengthLimit returns correct limit for Windows Vista"() { + given: "Windows OS at major version 6" + + String os = "Windows" + String version = "6.0.0" + + when: "the command length limit is queried" + + int limit = GenerateProtoTask.getCmdLengthLimit(os, version) + + then: "it returns the Vista limit" + limit == GenerateProtoTask.VISTA_CMD_LENGTH_LIMIT + } + + void "test getCmdLengthLimit returns correct limit for non-Windows OS"() { + given: "MacOS X at major version 10" + + String os = "Mac OS X" + String version = "10.0.0" + + when: "the command length limit is queried" + + int limit = GenerateProtoTask.getCmdLengthLimit(os, version) + + then: "it returns maximum integer value" + limit == Integer.MAX_VALUE + } }