diff --git a/README.md b/README.md index b0282fd..d728788 100644 --- a/README.md +++ b/README.md @@ -32,39 +32,47 @@ libraryDependencies += "org.apache.avro" % "avro" % avroCompilerVersion ## Settings -| Name | Default | Description | -|:-------------------------------------------|:-------------------------------------------|:------------| -| `avroSource` | `sourceDirectory` / `avro` | Source directory with `*.avsc`, `*.avdl` and `*.avpr` files. | -| `avroSchemaParserBuilder` | `DefaultSchemaParserBuilder.default()` | `.avsc` schema parser builder | -| `avroUnpackDependencies` / `includeFilter` | All avro specifications | Avro specification files from dependencies to unpack | -| `avroUnpackDependencies` / `excludeFilter` | Hidden files | Avro specification files from dependencies to exclude from unpacking | -| `avroUnpackDependencies` / `target` | `target` / `avro` / `$config` | Target directory for schemas packaged in the dependencies | -| `avroGenerate` / `target` | `target` / `compiled_avro` / `$config` | Source directory for generated `.java` files. | -| `avroDependencyIncludeFilter` | `source` typed `avro` classifier artifacts | Dependencies containing avro schema to be unpacked for generation | -| `avroIncludes` | `Seq()` | Paths with extra `*.avsc` files to be included in compilation. | -| `packageAvro` / `artifactClassifier` | `Some("avro")` | Classifier for avro artifact | -| `packageAvro` / `publishArtifact` | `false` | Enable / Disable avro artifact publishing | -| `avroStringType` | `CharSequence` | Type for representing strings. Possible values: `CharSequence`, `String`, `Utf8`. | -| `avroUseNamespace` | `false` | Validate that directory layout reflects namespaces, i.e. `com/myorg/MyRecord.avsc`. | -| `avroFieldVisibility` | `public` | Field Visibility for the properties. Possible values: `private`, `public`. | -| `avroEnableDecimalLogicalType` | `true` | Set to true to use `java.math.BigDecimal` instead of `java.nio.ByteBuffer` for logical type `decimal`. | -| `avroOptionalGetters` | `false` (requires avro `1.10+`) | Set to true to generate getters that return `Optional` for nullable fields. | +| Name | Default | Description | +|:-------------------------------------------|:-------------------------------------------|:----------------------------------------------------------------------------------------| +| `avroSource` | `sourceDirectory` / `avro` | Source directory with `*.avsc`, `*.avdl` and `*.avpr` files. | +| `avroSpecificRecords` | `Seq.empty` | List of avro generated classes to recompile with current avro version and settings. | +| `avroSchemaParserBuilder` | `DefaultSchemaParserBuilder.default()` | `.avsc` schema parser builder | +| `avroUnpackDependencies` / `includeFilter` | All avro specifications | Avro specification files from dependencies to unpack | +| `avroUnpackDependencies` / `excludeFilter` | Hidden files | Avro specification files from dependencies to exclude from unpacking | +| `avroUnpackDependencies` / `target` | `target` / `avro` / `$config` | Target directory for schemas packaged in the dependencies | +| `avroGenerate` / `target` | `target` / `compiled_avro` / `$config` | Source directory for generated `.java` files. | +| `avroDependencyIncludeFilter` | `source` typed `avro` classifier artifacts | Dependencies containing avro schema to be unpacked for generation | +| `avroIncludes` | `Seq()` | Paths with extra `*.avsc` files to be included in compilation. | +| `packageAvro` / `artifactClassifier` | `Some("avro")` | Classifier for avro artifact | +| `packageAvro` / `publishArtifact` | `false` | Enable / Disable avro artifact publishing | +| `avroStringType` | `CharSequence` | Type for representing strings. Possible values: `CharSequence`, `String`, `Utf8`. | +| `avroUseNamespace` | `false` | Validate that directory layout reflects namespaces, i.e. `com/myorg/MyRecord.avsc`. | +| `avroFieldVisibility` | `public` | Field Visibility for the properties. Possible values: `private`, `public`. | +| `avroEnableDecimalLogicalType` | `true` | Use `java.math.BigDecimal` instead of `java.nio.ByteBuffer` for logical type `decimal`. | +| `avroOptionalGetters` | `false` (requires avro `1.10+`) | Generate getters that return `Optional` for nullable fields. | + +## Tasks + +| Name | Description | +|:-------------------------|:--------------------------------------------------------------------------------------------------| +| `avroUnpackDependencies` | Unpack avro schemas from dependencies. This task is automatically executed before `avroGenerate`. | +| `avroGenerate` | Generate Java sources for Avro schemas. This task is automatically executed before `compile`. | +| `packageAvro` | Produces an avro artifact, such as a jar containing avro schemas. | ## Examples For example, to change the Java type of the string fields, add the following lines to `build.sbt`: -``` +```sbt avroStringType := "String" ``` -## Tasks +If you depend on an artifact with previously generated avro java classes with string fields as `CharSequence`, +you can recompile them with `String` by also adding the following -| Name | Description | -|:-------------------------|:------------| -| `avroUnpackDependencies` | Unpack avro schemas from dependencies. This task is automatically executed before `avroGenerate`. -| `avroGenerate` | Generate Java sources for Avro schemas. This task is automatically executed before `compile`. -| `packageAvro` | Produces an avro artifact, such as a jar containing avro schemas. +```sbt +Compile / avroSpecificRecords += classOf[com.example.MyAvroRecord] // lib must be declared in project/plugins.sbt +``` ## Packaging Avro files diff --git a/src/main/java/com/github/sbt/avro/mojo/AvscFilesCompiler.java b/src/main/java/com/github/sbt/avro/mojo/AvscFilesCompiler.java index 3480e5f..d104833 100644 --- a/src/main/java/com/github/sbt/avro/mojo/AvscFilesCompiler.java +++ b/src/main/java/com/github/sbt/avro/mojo/AvscFilesCompiler.java @@ -4,6 +4,8 @@ import org.apache.avro.SchemaParseException; import org.apache.avro.compiler.specific.SpecificCompiler; import org.apache.avro.generic.GenericData; +import org.apache.avro.specific.SpecificData; +import org.apache.avro.specific.SpecificRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,6 +75,47 @@ public void compileFiles(Set files, File outputDirectory) { } } + public void compileClasses(Set> classes, File outputDirectory) { + Set> compiledClasses = new HashSet<>(); + Set> uncompiledClasses = new HashSet<>(classes); + + boolean progressed = true; + while (progressed && !uncompiledClasses.isEmpty()) { + progressed = false; + compileExceptions = new HashMap<>(); + + for (Class clazz : uncompiledClasses) { + Schema schema = SpecificData.get().getSchema(clazz); + boolean success = tryCompile(null, schema, outputDirectory); + if (success) { + compiledClasses.add(clazz); + progressed = true; + } + } + + uncompiledClasses.removeAll(compiledClasses); + } + + if (!uncompiledClasses.isEmpty()) { + String failedFiles = uncompiledClasses.stream() + .map(Class::toString) + .collect(Collectors.joining(", ")); + SchemaGenerationException ex = new SchemaGenerationException( + String.format("Can not re-compile class: %s", failedFiles)); + + for (Class clazz : uncompiledClasses) { + Exception e = compileExceptions.get(clazz); + if (e != null) { + if (logCompileExceptions) { + LOG.error(clazz.toString(), e); + } + ex.addSuppressed(e); + } + } + throw ex; + } + } + private boolean tryCompile(AvroFileRef src, File outputDirectory) { Schema.Parser successfulSchemaParser = stashParser(); final Schema schema; @@ -87,6 +130,10 @@ private boolean tryCompile(AvroFileRef src, File outputDirectory) { throw new SchemaGenerationException(String.format("Error parsing schema file %s", src), e); } + return tryCompile(src.getFile(), schema, outputDirectory); + } + + private boolean tryCompile(File src, Schema schema, File outputDirectory) { SpecificCompiler compiler = new SpecificCompiler(schema); compiler.setTemplateDir(templateDirectory); compiler.setStringType(stringType); @@ -99,10 +146,10 @@ private boolean tryCompile(AvroFileRef src, File outputDirectory) { } try { - compiler.compileToDestination(src.getFile(), outputDirectory); + compiler.compileToDestination(src, outputDirectory); } catch (IOException e) { throw new SchemaGenerationException( - String.format("Error compiling schema file %s to %s", src, outputDirectory), e); + String.format("Error compiling schema file %s to %s", src, outputDirectory), e); } return true; diff --git a/src/main/scala/com/github/sbt/avro/SbtAvro.scala b/src/main/scala/com/github/sbt/avro/SbtAvro.scala index beec778..1e8142d 100644 --- a/src/main/scala/com/github/sbt/avro/SbtAvro.scala +++ b/src/main/scala/com/github/sbt/avro/SbtAvro.scala @@ -9,11 +9,13 @@ import sbt.Keys._ import sbt._ import CrossVersion.partialVersion import Path.relativeTo -import com.github.sbt.avro.mojo.{AvroFileRef, SchemaParserBuilder} +import com.github.sbt.avro.mojo.{AvroFileRef, AvscFilesCompiler, SchemaParserBuilder} +import org.apache.avro.specific.SpecificRecord import sbt.librarymanagement.DependencyFilter import java.io.File import java.util.jar.JarFile +import scala.collection.JavaConverters._ /** * Simple plugin for generating the Java sources for Avro schemas and protocols. @@ -46,17 +48,18 @@ object SbtAvro extends AutoPlugin { lazy val avroCompilerVersion: String = implVersion.getOrElse(bundleVersion) // format: off - val avroStringType = settingKey[String]("Type for representing strings. Possible values: CharSequence, String, Utf8. Default: CharSequence.") - val avroEnableDecimalLogicalType = settingKey[Boolean]("Set to true to use java.math.BigDecimal instead of java.nio.ByteBuffer for logical type \"decimal\".") + val avroCreateSetters = settingKey[Boolean]("Generate setters. Default: true") + val avroDependencyIncludeFilter = settingKey[DependencyFilter]("Filter for including modules containing avro dependencies.") + val avroEnableDecimalLogicalType = settingKey[Boolean]("Use java.math.BigDecimal instead of java.nio.ByteBuffer for logical type \"decimal\". Default: true.") val avroFieldVisibility = settingKey[String]("Field visibility for the properties. Possible values: private, public. Default: public.") - val avroUseNamespace = settingKey[Boolean]("Validate that directory layout reflects namespaces, i.e. src/main/avro/com/myorg/MyRecord.avsc.") - val avroOptionalGetters = settingKey[Boolean]("Set to true to generate getters that return Optional for nullable fields") - val avroCreateSetters = settingKey[Boolean]("Set to false to not generate setters. Default: true") - val avroSource = settingKey[File]("Default Avro source directory.") val avroIncludes = settingKey[Seq[File]]("Avro schema includes.") + val avroOptionalGetters = settingKey[Boolean]("Generate getters that return Optional for nullable fields. Default: false.") + val avroSpecificRecords = settingKey[Seq[Class[_ <: SpecificRecord]]]("List of avro records to recompile with current avro version and settings.") val avroSchemaParserBuilder = settingKey[SchemaParserBuilder](".avsc schema parser builder") + val avroSource = settingKey[File]("Default Avro source directory.") + val avroStringType = settingKey[String]("Type for representing strings. Possible values: CharSequence, String, Utf8. Default: CharSequence.") val avroUnpackDependencies = taskKey[Seq[File]]("Unpack avro dependencies.") - val avroDependencyIncludeFilter = settingKey[DependencyFilter]("Filter for including modules containing avro dependencies.") + val avroUseNamespace = settingKey[Boolean]("Validate that directory layout reflects namespaces, i.e. src/main/avro/com/myorg/MyRecord.avsc. Default: false.") val avroGenerate = taskKey[Seq[File]]("Generate Java sources for Avro schemas.") val packageAvro = taskKey[File]("Produces an avro artifact, such as a jar containing avro schemas.") @@ -78,6 +81,7 @@ object SbtAvro extends AutoPlugin { // settings to be applied for both Compile and Test lazy val configScopedSettings: Seq[Setting[_]] = Seq( avroSource := sourceDirectory.value / "avro", + avroSpecificRecords := Seq.empty, // dependencies avroUnpackDependencies / includeFilter := AllPassFilter, avroUnpackDependencies / excludeFilter := HiddenFileFilter, @@ -169,7 +173,44 @@ object SbtAvro extends AutoPlugin { ) } - def compileIdls(idls: Seq[File], target: File, log: Logger, stringType: StringType, fieldVisibility: FieldVisibility, enableDecimalLogicalType: Boolean, optionalGetters: Option[Boolean], createSetters: Boolean) = { + def recompile( + records: Seq[Class[_ <: SpecificRecord]], + target: File, + log: Logger, + stringType: StringType, + fieldVisibility: FieldVisibility, + enableDecimalLogicalType: Boolean, + useNamespace: Boolean, + optionalGetters: Option[Boolean], + createSetters: Boolean, + builder: SchemaParserBuilder + ) = { + val compiler = new AvscFilesCompiler(builder) + compiler.setStringType(stringType) + compiler.setFieldVisibility(fieldVisibility) + compiler.setUseNamespace(useNamespace) + compiler.setEnableDecimalLogicalType(enableDecimalLogicalType) + compiler.setCreateSetters(createSetters) + optionalGetters.foreach(compiler.setOptionalGetters) + compiler.setLogCompileExceptions(true) + compiler.setTemplateDirectory("/org/apache/avro/compiler/specific/templates/java/classic/") + + records.foreach { avsc => + log.info(s"Compiling Avro schemas $avsc") + } + compiler.compileClasses(records.toSet.asJava, target) + } + + def compileIdls( + idls: Seq[File], + target: File, + log: Logger, + stringType: StringType, + fieldVisibility: FieldVisibility, + enableDecimalLogicalType: Boolean, + optionalGetters: Option[Boolean], + createSetters: Boolean + ) = { idls.foreach { idl => log.info(s"Compiling Avro IDL $idl") val parser = new Idl(idl) @@ -185,10 +226,18 @@ object SbtAvro extends AutoPlugin { } } - def compileAvscs(refs: Seq[AvroFileRef], target: File, log: Logger, stringType: StringType, fieldVisibility: FieldVisibility, enableDecimalLogicalType: Boolean, useNamespace: Boolean, optionalGetters: Option[Boolean], createSetters: Boolean, builder: SchemaParserBuilder) = { - import com.github.sbt.avro.mojo._ - - import scala.collection.JavaConverters._ + def compileAvscs( + refs: Seq[AvroFileRef], + target: File, + log: Logger, + stringType: StringType, + fieldVisibility: FieldVisibility, + enableDecimalLogicalType: Boolean, + useNamespace: Boolean, + optionalGetters: Option[Boolean], + createSetters: Boolean, + builder: SchemaParserBuilder + ) = { val compiler = new AvscFilesCompiler(builder) compiler.setStringType(stringType) compiler.setFieldVisibility(fieldVisibility) @@ -205,7 +254,16 @@ object SbtAvro extends AutoPlugin { compiler.compileFiles(refs.toSet.asJava, target) } - def compileAvprs(avprs: Seq[File], target: File, log: Logger, stringType: StringType, fieldVisibility: FieldVisibility, enableDecimalLogicalType: Boolean, optionalGetters: Option[Boolean], createSetters: Boolean) = { + def compileAvprs( + avprs: Seq[File], + target: File, + log: Logger, + stringType: StringType, + fieldVisibility: FieldVisibility, + enableDecimalLogicalType: Boolean, + optionalGetters: Option[Boolean], + createSetters: Boolean + ) = { avprs.foreach { avpr => log.info(s"Compiling Avro protocol $avpr") val protocol = Protocol.parse(avpr) @@ -220,20 +278,24 @@ object SbtAvro extends AutoPlugin { } } - private[this] def compileAvroSchema(srcDirs: Seq[File], - target: File, - log: Logger, - stringType: StringType, - fieldVisibility: FieldVisibility, - enableDecimalLogicalType: Boolean, - useNamespace: Boolean, - optionalGetters: Option[Boolean], - createSetters: Boolean, - builder: SchemaParserBuilder): Set[File] = { + private[this] def compileAvroSchema( + records: Seq[Class[_ <: SpecificRecord]], + srcDirs: Seq[File], + target: File, + log: Logger, + stringType: StringType, + fieldVisibility: FieldVisibility, + enableDecimalLogicalType: Boolean, + useNamespace: Boolean, + optionalGetters: Option[Boolean], + createSetters: Boolean, + builder: SchemaParserBuilder + ): Set[File] = { val avdls = srcDirs.flatMap(d => (d ** AvroAvdlFilter).get) val avscs = srcDirs.flatMap(d => (d ** AvroAvscFilter).get.map(avsc => new AvroFileRef(d, avsc.relativeTo(d).get.toString))) val avprs = srcDirs.flatMap(d => (d ** AvroAvrpFilter).get) + recompile(records, target, log, stringType, fieldVisibility, enableDecimalLogicalType, useNamespace, optionalGetters, createSetters, builder) compileIdls(avdls, target, log, stringType, fieldVisibility, enableDecimalLogicalType, optionalGetters, createSetters) compileAvscs(avscs, target, log, stringType, fieldVisibility, enableDecimalLogicalType, useNamespace, optionalGetters, createSetters, builder) compileAvprs(avprs, target, log, stringType, fieldVisibility, enableDecimalLogicalType, optionalGetters, createSetters) @@ -242,8 +304,8 @@ object SbtAvro extends AutoPlugin { } private def sourceGeneratorTask(key: TaskKey[Seq[File]]) = Def.task { - val out = (key / streams).value + val records = avroSpecificRecords.value val srcDir = avroSource.value val externalSrcDir = (avroUnpackDependencies / target).value val includes = avroIncludes.value @@ -262,7 +324,19 @@ object SbtAvro extends AutoPlugin { val cachedCompile = { FileFunction.cached(out.cacheDirectory / "avro", FilesInfo.lastModified, FilesInfo.exists) { _ => out.log.info(s"Avro compiler $avroCompilerVersion using stringType=$strType") - compileAvroSchema(srcDirs, outDir, out.log, strType, fieldVis, enbDecimal, useNs, optionalGetters, createSetters, builder) + compileAvroSchema( + records, + srcDirs, + outDir, + out.log, + strType, + fieldVis, + enbDecimal, + useNs, + optionalGetters, + createSetters, + builder + ) } } diff --git a/src/sbt-test/sbt-avro/settings/build.sbt b/src/sbt-test/sbt-avro/settings/build.sbt index a43b9ed..4b6453c 100644 --- a/src/sbt-test/sbt-avro/settings/build.sbt +++ b/src/sbt-test/sbt-avro/settings/build.sbt @@ -8,5 +8,7 @@ libraryDependencies ++= Seq( avroStringType := "String" avroFieldVisibility := "public" avroOptionalGetters := true -(Compile / avroSource) := (Compile / sourceDirectory).value / "avro_source" -(Compile / avroGenerate / target) := (Compile / sourceManaged).value +avroEnableDecimalLogicalType := false +Compile / avroSpecificRecords += classOf[org.apache.avro.specific.TestRecordWithLogicalTypes] +Compile / avroSource := (Compile / sourceDirectory).value / "avro_source" +Compile / avroGenerate / target := (Compile / sourceManaged).value diff --git a/src/sbt-test/sbt-avro/settings/project/plugins.sbt b/src/sbt-test/sbt-avro/settings/project/plugins.sbt index 92501b2..40bb6b3 100644 --- a/src/sbt-test/sbt-avro/settings/project/plugins.sbt +++ b/src/sbt-test/sbt-avro/settings/project/plugins.sbt @@ -4,4 +4,8 @@ sys.props.get("plugin.version") match { |Specify this property using the scriptedLaunchOpts -D.""".stripMargin) } -libraryDependencies += "org.apache.avro" % "avro-compiler" % "1.11.2" +libraryDependencies ++= Seq( + "org.apache.avro" % "avro-compiler" % "1.11.2", + // depend on test jar to get some generated records in the build + "org.apache.avro" % "avro" % "1.11.2" classifier "tests" +) diff --git a/src/sbt-test/sbt-avro/settings/test b/src/sbt-test/sbt-avro/settings/test index f94deeb..59f1cd9 100644 --- a/src/sbt-test/sbt-avro/settings/test +++ b/src/sbt-test/sbt-avro/settings/test @@ -1,5 +1,9 @@ > compile +$ exists target/scala-2.13/src_managed/main/org/apache/avro/specific/TestRecordWithLogicalTypes.java +-$ exec grep BigDecimal target/scala-2.13/src_managed/main/org/apache/avro/specific/TestRecordWithLogicalTypes.java +-$ exec grep CharSequence target/scala-2.13/src_managed/main/org/apache/avro/specific/TestRecordWithLogicalTypes.java + $ exists target/scala-2.13/src_managed/main/com/github/sbt/avro/test/settings/Avdl.java $ exists target/scala-2.13/src_managed/main/com/github/sbt/avro/test/settings/Avpr.java $ exists target/scala-2.13/src_managed/main/com/github/sbt/avro/test/settings/ProtocolAvpr.java diff --git a/src/test/java/com/github/sbt/avro/test/TestSpecificRecord.java b/src/test/java/com/github/sbt/avro/test/TestSpecificRecord.java new file mode 100644 index 0000000..39f6d7f --- /dev/null +++ b/src/test/java/com/github/sbt/avro/test/TestSpecificRecord.java @@ -0,0 +1,308 @@ +/** + * Autogenerated by Avro + * + * DO NOT EDIT DIRECTLY + */ +package com.github.sbt.avro.test; + +import org.apache.avro.specific.SpecificData; +import org.apache.avro.util.Utf8; +import org.apache.avro.message.BinaryMessageEncoder; +import org.apache.avro.message.BinaryMessageDecoder; +import org.apache.avro.message.SchemaStore; + +@org.apache.avro.specific.AvroGenerated +public class TestSpecificRecord extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord { + private static final long serialVersionUID = 8383833071851424655L; + + + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"TestSpecificRecord\",\"namespace\":\"com.github.sbt.avro\",\"fields\":[{\"name\":\"value\",\"type\":\"string\"}]}"); + public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } + + private static final SpecificData MODEL$ = new SpecificData(); + + private static final BinaryMessageEncoder ENCODER = + new BinaryMessageEncoder<>(MODEL$, SCHEMA$); + + private static final BinaryMessageDecoder DECODER = + new BinaryMessageDecoder<>(MODEL$, SCHEMA$); + + /** + * Return the BinaryMessageEncoder instance used by this class. + * @return the message encoder used by this class + */ + public static BinaryMessageEncoder getEncoder() { + return ENCODER; + } + + /** + * Return the BinaryMessageDecoder instance used by this class. + * @return the message decoder used by this class + */ + public static BinaryMessageDecoder getDecoder() { + return DECODER; + } + + /** + * Create a new BinaryMessageDecoder instance for this class that uses the specified {@link SchemaStore}. + * @param resolver a {@link SchemaStore} used to find schemas by fingerprint + * @return a BinaryMessageDecoder instance for this class backed by the given SchemaStore + */ + public static BinaryMessageDecoder createDecoder(SchemaStore resolver) { + return new BinaryMessageDecoder<>(MODEL$, SCHEMA$, resolver); + } + + /** + * Serializes this TestSpecificRecord to a ByteBuffer. + * @return a buffer holding the serialized data for this instance + * @throws java.io.IOException if this instance could not be serialized + */ + public java.nio.ByteBuffer toByteBuffer() throws java.io.IOException { + return ENCODER.encode(this); + } + + /** + * Deserializes a TestSpecificRecord from a ByteBuffer. + * @param b a byte buffer holding serialized data for an instance of this class + * @return a TestSpecificRecord instance decoded from the given buffer + * @throws java.io.IOException if the given bytes could not be deserialized into an instance of this class + */ + public static TestSpecificRecord fromByteBuffer( + java.nio.ByteBuffer b) throws java.io.IOException { + return DECODER.decode(b); + } + + private java.lang.CharSequence value; + + /** + * Default constructor. Note that this does not initialize fields + * to their default values from the schema. If that is desired then + * one should use newBuilder(). + */ + public TestSpecificRecord() {} + + /** + * All-args constructor. + * @param value The new value for value + */ + public TestSpecificRecord(java.lang.CharSequence value) { + this.value = value; + } + + @Override + public org.apache.avro.specific.SpecificData getSpecificData() { return MODEL$; } + + @Override + public org.apache.avro.Schema getSchema() { return SCHEMA$; } + + // Used by DatumWriter. Applications should not call. + @Override + public java.lang.Object get(int field$) { + switch (field$) { + case 0: return value; + default: throw new IndexOutOfBoundsException("Invalid index: " + field$); + } + } + + // Used by DatumReader. Applications should not call. + @Override + @SuppressWarnings(value="unchecked") + public void put(int field$, java.lang.Object value$) { + switch (field$) { + case 0: value = (java.lang.CharSequence)value$; break; + default: throw new IndexOutOfBoundsException("Invalid index: " + field$); + } + } + + /** + * Gets the value of the 'value' field. + * @return The value of the 'value' field. + */ + public java.lang.CharSequence getValue() { + return value; + } + + + /** + * Sets the value of the 'value' field. + * @param value the value to set. + */ + public void setValue(java.lang.CharSequence value) { + this.value = value; + } + + /** + * Creates a new TestSpecificRecord RecordBuilder. + * @return A new TestSpecificRecord RecordBuilder + */ + public static TestSpecificRecord.Builder newBuilder() { + return new TestSpecificRecord.Builder(); + } + + /** + * Creates a new TestSpecificRecord RecordBuilder by copying an existing Builder. + * @param other The existing builder to copy. + * @return A new TestSpecificRecord RecordBuilder + */ + public static TestSpecificRecord.Builder newBuilder(TestSpecificRecord.Builder other) { + if (other == null) { + return new TestSpecificRecord.Builder(); + } else { + return new TestSpecificRecord.Builder(other); + } + } + + /** + * Creates a new TestSpecificRecord RecordBuilder by copying an existing TestSpecificRecord instance. + * @param other The existing instance to copy. + * @return A new TestSpecificRecord RecordBuilder + */ + public static TestSpecificRecord.Builder newBuilder(TestSpecificRecord other) { + if (other == null) { + return new TestSpecificRecord.Builder(); + } else { + return new TestSpecificRecord.Builder(other); + } + } + + /** + * RecordBuilder for TestSpecificRecord instances. + */ + @org.apache.avro.specific.AvroGenerated + public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase + implements org.apache.avro.data.RecordBuilder { + + private java.lang.CharSequence value; + + /** Creates a new Builder */ + private Builder() { + super(SCHEMA$, MODEL$); + } + + /** + * Creates a Builder by copying an existing Builder. + * @param other The existing Builder to copy. + */ + private Builder(TestSpecificRecord.Builder other) { + super(other); + if (isValidValue(fields()[0], other.value)) { + this.value = data().deepCopy(fields()[0].schema(), other.value); + fieldSetFlags()[0] = other.fieldSetFlags()[0]; + } + } + + /** + * Creates a Builder by copying an existing TestSpecificRecord instance + * @param other The existing instance to copy. + */ + private Builder(TestSpecificRecord other) { + super(SCHEMA$, MODEL$); + if (isValidValue(fields()[0], other.value)) { + this.value = data().deepCopy(fields()[0].schema(), other.value); + fieldSetFlags()[0] = true; + } + } + + /** + * Gets the value of the 'value' field. + * @return The value. + */ + public java.lang.CharSequence getValue() { + return value; + } + + + /** + * Sets the value of the 'value' field. + * @param value The value of 'value'. + * @return This builder. + */ + public TestSpecificRecord.Builder setValue(java.lang.CharSequence value) { + validate(fields()[0], value); + this.value = value; + fieldSetFlags()[0] = true; + return this; + } + + /** + * Checks whether the 'value' field has been set. + * @return True if the 'value' field has been set, false otherwise. + */ + public boolean hasValue() { + return fieldSetFlags()[0]; + } + + + /** + * Clears the value of the 'value' field. + * @return This builder. + */ + public TestSpecificRecord.Builder clearValue() { + value = null; + fieldSetFlags()[0] = false; + return this; + } + + @Override + @SuppressWarnings("unchecked") + public TestSpecificRecord build() { + try { + TestSpecificRecord record = new TestSpecificRecord(); + record.value = fieldSetFlags()[0] ? this.value : (java.lang.CharSequence) defaultValue(fields()[0]); + return record; + } catch (org.apache.avro.AvroMissingFieldException e) { + throw e; + } catch (java.lang.Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } + } + + @SuppressWarnings("unchecked") + private static final org.apache.avro.io.DatumWriter + WRITER$ = (org.apache.avro.io.DatumWriter)MODEL$.createDatumWriter(SCHEMA$); + + @Override public void writeExternal(java.io.ObjectOutput out) + throws java.io.IOException { + WRITER$.write(this, SpecificData.getEncoder(out)); + } + + @SuppressWarnings("unchecked") + private static final org.apache.avro.io.DatumReader + READER$ = (org.apache.avro.io.DatumReader)MODEL$.createDatumReader(SCHEMA$); + + @Override public void readExternal(java.io.ObjectInput in) + throws java.io.IOException { + READER$.read(this, SpecificData.getDecoder(in)); + } + + @Override protected boolean hasCustomCoders() { return true; } + + @Override public void customEncode(org.apache.avro.io.Encoder out) + throws java.io.IOException + { + out.writeString(this.value); + + } + + @Override public void customDecode(org.apache.avro.io.ResolvingDecoder in) + throws java.io.IOException + { + org.apache.avro.Schema.Field[] fieldOrder = in.readFieldOrderIfDiff(); + if (fieldOrder == null) { + this.value = in.readString(this.value instanceof Utf8 ? (Utf8)this.value : null); + + } else { + for (int i = 0; i < 1; i++) { + switch (fieldOrder[i].pos()) { + case 0: + this.value = in.readString(this.value instanceof Utf8 ? (Utf8)this.value : null); + break; + + default: + throw new java.io.IOException("Corrupt ResolvingDecoder."); + } + } + } + } +} \ No newline at end of file diff --git a/src/test/java/com/github/sbt/avro/test/TestSpecificRecordParent.java b/src/test/java/com/github/sbt/avro/test/TestSpecificRecordParent.java new file mode 100644 index 0000000..d3331e6 --- /dev/null +++ b/src/test/java/com/github/sbt/avro/test/TestSpecificRecordParent.java @@ -0,0 +1,365 @@ +/** + * Autogenerated by Avro + * + * DO NOT EDIT DIRECTLY + */ +package com.github.sbt.avro.test; + +import org.apache.avro.generic.GenericArray; +import org.apache.avro.specific.SpecificData; +import org.apache.avro.util.Utf8; +import org.apache.avro.message.BinaryMessageEncoder; +import org.apache.avro.message.BinaryMessageDecoder; +import org.apache.avro.message.SchemaStore; + +@org.apache.avro.specific.AvroGenerated +public class TestSpecificRecordParent extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord { + private static final long serialVersionUID = 7223714509976921291L; + + + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"TestSpecificRecordParent\",\"namespace\":\"com.github.sbt.avro\",\"fields\":[{\"name\":\"child\",\"type\":{\"type\":\"record\",\"name\":\"TestSpecificRecord\",\"fields\":[{\"name\":\"value\",\"type\":\"string\"}]}}]}"); + public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } + + private static final SpecificData MODEL$ = new SpecificData(); + + private static final BinaryMessageEncoder ENCODER = + new BinaryMessageEncoder<>(MODEL$, SCHEMA$); + + private static final BinaryMessageDecoder DECODER = + new BinaryMessageDecoder<>(MODEL$, SCHEMA$); + + /** + * Return the BinaryMessageEncoder instance used by this class. + * @return the message encoder used by this class + */ + public static BinaryMessageEncoder getEncoder() { + return ENCODER; + } + + /** + * Return the BinaryMessageDecoder instance used by this class. + * @return the message decoder used by this class + */ + public static BinaryMessageDecoder getDecoder() { + return DECODER; + } + + /** + * Create a new BinaryMessageDecoder instance for this class that uses the specified {@link SchemaStore}. + * @param resolver a {@link SchemaStore} used to find schemas by fingerprint + * @return a BinaryMessageDecoder instance for this class backed by the given SchemaStore + */ + public static BinaryMessageDecoder createDecoder(SchemaStore resolver) { + return new BinaryMessageDecoder<>(MODEL$, SCHEMA$, resolver); + } + + /** + * Serializes this TestSpecificRecordParent to a ByteBuffer. + * @return a buffer holding the serialized data for this instance + * @throws java.io.IOException if this instance could not be serialized + */ + public java.nio.ByteBuffer toByteBuffer() throws java.io.IOException { + return ENCODER.encode(this); + } + + /** + * Deserializes a TestSpecificRecordParent from a ByteBuffer. + * @param b a byte buffer holding serialized data for an instance of this class + * @return a TestSpecificRecordParent instance decoded from the given buffer + * @throws java.io.IOException if the given bytes could not be deserialized into an instance of this class + */ + public static TestSpecificRecordParent fromByteBuffer( + java.nio.ByteBuffer b) throws java.io.IOException { + return DECODER.decode(b); + } + + private com.github.sbt.avro.test.TestSpecificRecord child; + + /** + * Default constructor. Note that this does not initialize fields + * to their default values from the schema. If that is desired then + * one should use newBuilder(). + */ + public TestSpecificRecordParent() {} + + /** + * All-args constructor. + * @param child The new value for child + */ + public TestSpecificRecordParent(com.github.sbt.avro.test.TestSpecificRecord child) { + this.child = child; + } + + @Override + public org.apache.avro.specific.SpecificData getSpecificData() { return MODEL$; } + + @Override + public org.apache.avro.Schema getSchema() { return SCHEMA$; } + + // Used by DatumWriter. Applications should not call. + @Override + public java.lang.Object get(int field$) { + switch (field$) { + case 0: return child; + default: throw new IndexOutOfBoundsException("Invalid index: " + field$); + } + } + + // Used by DatumReader. Applications should not call. + @Override + @SuppressWarnings(value="unchecked") + public void put(int field$, java.lang.Object value$) { + switch (field$) { + case 0: child = (com.github.sbt.avro.test.TestSpecificRecord)value$; break; + default: throw new IndexOutOfBoundsException("Invalid index: " + field$); + } + } + + /** + * Gets the value of the 'child' field. + * @return The value of the 'child' field. + */ + public com.github.sbt.avro.test.TestSpecificRecord getChild() { + return child; + } + + + /** + * Sets the value of the 'child' field. + * @param value the value to set. + */ + public void setChild(com.github.sbt.avro.test.TestSpecificRecord value) { + this.child = value; + } + + /** + * Creates a new TestSpecificRecordParent RecordBuilder. + * @return A new TestSpecificRecordParent RecordBuilder + */ + public static com.github.sbt.avro.test.TestSpecificRecordParent.Builder newBuilder() { + return new com.github.sbt.avro.test.TestSpecificRecordParent.Builder(); + } + + /** + * Creates a new TestSpecificRecordParent RecordBuilder by copying an existing Builder. + * @param other The existing builder to copy. + * @return A new TestSpecificRecordParent RecordBuilder + */ + public static com.github.sbt.avro.test.TestSpecificRecordParent.Builder newBuilder(com.github.sbt.avro.test.TestSpecificRecordParent.Builder other) { + if (other == null) { + return new com.github.sbt.avro.test.TestSpecificRecordParent.Builder(); + } else { + return new com.github.sbt.avro.test.TestSpecificRecordParent.Builder(other); + } + } + + /** + * Creates a new TestSpecificRecordParent RecordBuilder by copying an existing TestSpecificRecordParent instance. + * @param other The existing instance to copy. + * @return A new TestSpecificRecordParent RecordBuilder + */ + public static com.github.sbt.avro.test.TestSpecificRecordParent.Builder newBuilder(com.github.sbt.avro.test.TestSpecificRecordParent other) { + if (other == null) { + return new com.github.sbt.avro.test.TestSpecificRecordParent.Builder(); + } else { + return new com.github.sbt.avro.test.TestSpecificRecordParent.Builder(other); + } + } + + /** + * RecordBuilder for TestSpecificRecordParent instances. + */ + @org.apache.avro.specific.AvroGenerated + public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase + implements org.apache.avro.data.RecordBuilder { + + private com.github.sbt.avro.test.TestSpecificRecord child; + private com.github.sbt.avro.test.TestSpecificRecord.Builder childBuilder; + + /** Creates a new Builder */ + private Builder() { + super(SCHEMA$, MODEL$); + } + + /** + * Creates a Builder by copying an existing Builder. + * @param other The existing Builder to copy. + */ + private Builder(com.github.sbt.avro.test.TestSpecificRecordParent.Builder other) { + super(other); + if (isValidValue(fields()[0], other.child)) { + this.child = data().deepCopy(fields()[0].schema(), other.child); + fieldSetFlags()[0] = other.fieldSetFlags()[0]; + } + if (other.hasChildBuilder()) { + this.childBuilder = com.github.sbt.avro.test.TestSpecificRecord.newBuilder(other.getChildBuilder()); + } + } + + /** + * Creates a Builder by copying an existing TestSpecificRecordParent instance + * @param other The existing instance to copy. + */ + private Builder(com.github.sbt.avro.test.TestSpecificRecordParent other) { + super(SCHEMA$, MODEL$); + if (isValidValue(fields()[0], other.child)) { + this.child = data().deepCopy(fields()[0].schema(), other.child); + fieldSetFlags()[0] = true; + } + this.childBuilder = null; + } + + /** + * Gets the value of the 'child' field. + * @return The value. + */ + public com.github.sbt.avro.test.TestSpecificRecord getChild() { + return child; + } + + + /** + * Sets the value of the 'child' field. + * @param value The value of 'child'. + * @return This builder. + */ + public com.github.sbt.avro.test.TestSpecificRecordParent.Builder setChild(com.github.sbt.avro.test.TestSpecificRecord value) { + validate(fields()[0], value); + this.childBuilder = null; + this.child = value; + fieldSetFlags()[0] = true; + return this; + } + + /** + * Checks whether the 'child' field has been set. + * @return True if the 'child' field has been set, false otherwise. + */ + public boolean hasChild() { + return fieldSetFlags()[0]; + } + + /** + * Gets the Builder instance for the 'child' field and creates one if it doesn't exist yet. + * @return This builder. + */ + public com.github.sbt.avro.test.TestSpecificRecord.Builder getChildBuilder() { + if (childBuilder == null) { + if (hasChild()) { + setChildBuilder(com.github.sbt.avro.test.TestSpecificRecord.newBuilder(child)); + } else { + setChildBuilder(com.github.sbt.avro.test.TestSpecificRecord.newBuilder()); + } + } + return childBuilder; + } + + /** + * Sets the Builder instance for the 'child' field + * @param value The builder instance that must be set. + * @return This builder. + */ + + public com.github.sbt.avro.test.TestSpecificRecordParent.Builder setChildBuilder(com.github.sbt.avro.test.TestSpecificRecord.Builder value) { + clearChild(); + childBuilder = value; + return this; + } + + /** + * Checks whether the 'child' field has an active Builder instance + * @return True if the 'child' field has an active Builder instance + */ + public boolean hasChildBuilder() { + return childBuilder != null; + } + + /** + * Clears the value of the 'child' field. + * @return This builder. + */ + public com.github.sbt.avro.test.TestSpecificRecordParent.Builder clearChild() { + child = null; + childBuilder = null; + fieldSetFlags()[0] = false; + return this; + } + + @Override + @SuppressWarnings("unchecked") + public TestSpecificRecordParent build() { + try { + TestSpecificRecordParent record = new TestSpecificRecordParent(); + if (childBuilder != null) { + try { + record.child = this.childBuilder.build(); + } catch (org.apache.avro.AvroMissingFieldException e) { + e.addParentField(record.getSchema().getField("child")); + throw e; + } + } else { + record.child = fieldSetFlags()[0] ? this.child : (com.github.sbt.avro.test.TestSpecificRecord) defaultValue(fields()[0]); + } + return record; + } catch (org.apache.avro.AvroMissingFieldException e) { + throw e; + } catch (java.lang.Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } + } + + @SuppressWarnings("unchecked") + private static final org.apache.avro.io.DatumWriter + WRITER$ = (org.apache.avro.io.DatumWriter)MODEL$.createDatumWriter(SCHEMA$); + + @Override public void writeExternal(java.io.ObjectOutput out) + throws java.io.IOException { + WRITER$.write(this, SpecificData.getEncoder(out)); + } + + @SuppressWarnings("unchecked") + private static final org.apache.avro.io.DatumReader + READER$ = (org.apache.avro.io.DatumReader)MODEL$.createDatumReader(SCHEMA$); + + @Override public void readExternal(java.io.ObjectInput in) + throws java.io.IOException { + READER$.read(this, SpecificData.getDecoder(in)); + } + + @Override protected boolean hasCustomCoders() { return true; } + + @Override public void customEncode(org.apache.avro.io.Encoder out) + throws java.io.IOException + { + this.child.customEncode(out); + + } + + @Override public void customDecode(org.apache.avro.io.ResolvingDecoder in) + throws java.io.IOException + { + org.apache.avro.Schema.Field[] fieldOrder = in.readFieldOrderIfDiff(); + if (fieldOrder == null) { + if (this.child == null) { + this.child = new com.github.sbt.avro.test.TestSpecificRecord(); + } + this.child.customDecode(in); + + } else { + for (int i = 0; i < 1; i++) { + switch (fieldOrder[i].pos()) { + case 0: + if (this.child == null) { + this.child = new com.github.sbt.avro.test.TestSpecificRecord(); + } + this.child.customDecode(in); + break; + + default: + throw new java.io.IOException("Corrupt ResolvingDecoder."); + } + } + } + } +} \ No newline at end of file diff --git a/src/test/resources/avro/test_records.avsc b/src/test/resources/avro/test_records.avsc new file mode 100644 index 0000000..b6a9c04 --- /dev/null +++ b/src/test/resources/avro/test_records.avsc @@ -0,0 +1,24 @@ +[ + { + "name": "TestSpecificRecord", + "namespace": "com.github.sbt.avro", + "type": "record", + "fields": [ + { + "name": "value", + "type": "string" + } + ] + }, + { + "name": "TestSpecificRecordParent", + "namespace": "com.github.sbt.avro", + "type": "record", + "fields": [ + { + "name": "child", + "type": "TestSpecificRecord" + } + ] + } +] \ No newline at end of file diff --git a/src/test/scala/com/github/sbt/avro/SbtAvroSpec.scala b/src/test/scala/com/github/sbt/avro/SbtAvroSpec.scala index 3018d1d..21b19b3 100644 --- a/src/test/scala/com/github/sbt/avro/SbtAvroSpec.scala +++ b/src/test/scala/com/github/sbt/avro/SbtAvroSpec.scala @@ -1,54 +1,54 @@ package com.github.sbt.avro -import java.io.File - import com.github.sbt.avro.mojo.AvroFileRef +import com.github.sbt.avro.test.{TestSpecificRecord, TestSpecificRecordParent} import org.apache.avro.compiler.specific.SpecificCompiler.FieldVisibility import org.apache.avro.generic.GenericData.StringType import org.specs2.mutable.Specification import sbt.util.Logger -/** - * Created by jeromewacongne on 06/08/2015. - */ +import java.io.File +import java.nio.file.Files + class SbtAvroSpec extends Specification { val builder = DefaultSchemaParserBuilder.default() val sourceDir = new File(getClass.getClassLoader.getResource("avro").toURI) - val targetDir = new File(sourceDir.getParentFile, "generated") - val logger = Logger.Null - val fullyQualifiedNames = Seq( - new File(sourceDir, "a.avsc"), - new File(sourceDir, "b.avsc"), - new File(sourceDir, "c.avsc"), - new File(sourceDir, "d.avsc"), - new File(sourceDir, "e.avsc")) - - val simpleNames = Seq( - new File(sourceDir, "_a.avsc"), - new File(sourceDir, "_b.avsc"), - new File(sourceDir, "_c.avsc"), - new File(sourceDir, "_d.avsc"), - new File(sourceDir, "_e.avsc")) - - val expectedOrderFullyQualifiedNames = Seq( - new File(sourceDir, "c.avsc"), - new File(sourceDir, "e.avsc"), - new File(sourceDir, "d.avsc"), - new File(sourceDir, "b.avsc"), - new File(sourceDir, "a.avsc")) - - val expectedOrderSimpleNames = Seq( - new File(sourceDir, "_c.avsc"), - new File(sourceDir, "_e.avsc"), - new File(sourceDir, "_d.avsc"), - new File(sourceDir, "_b.avsc"), - new File(sourceDir, "_a.avsc")) - - val sourceFiles = fullyQualifiedNames ++ simpleNames + + val targetDir = Files.createTempDirectory("sbt-avro").toFile + val packageDir = new File(targetDir, "com/github/sbt/avro/test") + val logger = Logger.Null "It should be possible to compile types depending on others if source files are provided in right order" >> { - val packageDir = new File(targetDir, "com/github/sbt/avro/test") + val fullyQualifiedNames = Seq( + new File(sourceDir, "a.avsc"), + new File(sourceDir, "b.avsc"), + new File(sourceDir, "c.avsc"), + new File(sourceDir, "d.avsc"), + new File(sourceDir, "e.avsc")) + + val simpleNames = Seq( + new File(sourceDir, "_a.avsc"), + new File(sourceDir, "_b.avsc"), + new File(sourceDir, "_c.avsc"), + new File(sourceDir, "_d.avsc"), + new File(sourceDir, "_e.avsc")) + + val expectedOrderFullyQualifiedNames = Seq( + new File(sourceDir, "c.avsc"), + new File(sourceDir, "e.avsc"), + new File(sourceDir, "d.avsc"), + new File(sourceDir, "b.avsc"), + new File(sourceDir, "a.avsc")) + + val expectedOrderSimpleNames = Seq( + new File(sourceDir, "_c.avsc"), + new File(sourceDir, "_e.avsc"), + new File(sourceDir, "_d.avsc"), + new File(sourceDir, "_b.avsc"), + new File(sourceDir, "_a.avsc")) + + val sourceFiles = fullyQualifiedNames ++ simpleNames val aJavaFile = new File(packageDir, "A.java") val bJavaFile = new File(packageDir, "B.java") @@ -99,7 +99,32 @@ class SbtAvroSpec extends Specification { _cJavaFile.isFile must beTrue _dJavaFile.isFile must beTrue _eJavaFile.isFile must beTrue + } + + "It should be possible to compile types depending on others if classes are provided in right order" >> { + // TestSpecificRecordParent and TestSpecificRecord were previously generated from test_records.avsc + SbtAvro.recompile( + records = Seq( + // put parent 1st + classOf[TestSpecificRecordParent], + classOf[TestSpecificRecord] + ), + target = targetDir, + log = logger, + stringType = StringType.CharSequence, + fieldVisibility = FieldVisibility.PRIVATE, + enableDecimalLogicalType = true, + useNamespace = false, + optionalGetters = None, + createSetters = true, + builder = builder + ) + + val record = new File(packageDir, "TestSpecificRecord.java") + val recordParent = new File(packageDir, "TestSpecificRecordParent.java") + record.isFile must beTrue + recordParent.isFile must beTrue } }