Skip to content

Commit

Permalink
Use single avsc schema builder
Browse files Browse the repository at this point in the history
  • Loading branch information
@RustedBones
RustedBones committed Sep 16, 2020
1 parent 1c0f4a7 commit b8b2cb9
Show file tree
Hide file tree
Showing 4 changed files with 41 additions and 54 deletions.
6 changes: 0 additions & 6 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,12 +90,6 @@ libraryDependencies += "org" % "name" % "rev" // module containing avro schemas
avroDependencyIncludeFilter := avroDependencyIncludeFilter.value || moduleFilter(organization = "org", name = "name")
```

Avro dependencies can be included in the compilation of your own `avsc` files:
```
avroIncludes := Seq((avroUnpackDependencies / target).value)
```
This means you can reference types defined in dependencies in your own avro schemas.

# License
This program is distributed under the BSD license. See the file `LICENSE` for more details.

Expand Down
82 changes: 37 additions & 45 deletions src/main/scala/sbtavro/SbtAvro.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,18 +135,22 @@ object SbtAvro extends AutoPlugin {
)
}

def compileIdl(idl: File, target: File, stringType: StringType, fieldVisibility: FieldVisibility, enableDecimalLogicalType: Boolean) {
val parser = new Idl(idl)
val protocol = Protocol.parse(parser.CompilationUnit.toString)
val compiler = new SpecificCompiler(protocol)
compiler.setStringType(stringType)
compiler.setFieldVisibility(fieldVisibility)
compiler.setEnableDecimalLogicalType(enableDecimalLogicalType)
compiler.compileToDestination(null, target)
def compileIdls(idls: Seq[File], target: File, log: Logger, stringType: StringType, fieldVisibility: FieldVisibility, enableDecimalLogicalType: Boolean) = {
idls.foreach { idl =>
log.info(s"Compiling Avro IDL $idl")
val parser = new Idl(idl)
val protocol = Protocol.parse(parser.CompilationUnit.toString)
val compiler = new SpecificCompiler(protocol)
compiler.setStringType(stringType)
compiler.setFieldVisibility(fieldVisibility)
compiler.setEnableDecimalLogicalType(enableDecimalLogicalType)
compiler.compileToDestination(null, target)
}
}

def compileAvscs(refs: Seq[AvroFileRef], target: File, stringType: StringType, fieldVisibility: FieldVisibility, enableDecimalLogicalType: Boolean, useNamespace: Boolean, builder: SchemaParserBuilder) {
def compileAvscs(refs: Seq[AvroFileRef], target: File, log: Logger, stringType: StringType, fieldVisibility: FieldVisibility, enableDecimalLogicalType: Boolean, useNamespace: Boolean, builder: SchemaParserBuilder) = {
import com.spotify.avro.mojo._
import scala.collection.JavaConverters._
val compiler = new AvscFilesCompiler(builder)
compiler.setStringType(stringType)
compiler.setFieldVisibility(fieldVisibility)
Expand All @@ -156,59 +160,49 @@ object SbtAvro extends AutoPlugin {
compiler.setLogCompileExceptions(true)
compiler.setTemplateDirectory("/org/apache/avro/compiler/specific/templates/java/classic/")

import scala.collection.JavaConverters._
refs.foreach { avsc =>
log.info(s"Compiling Avro schemas $avsc")
}
compiler.compileFiles(refs.toSet.asJava, target)
}

def compileAvpr(avpr: File, target: File, stringType: StringType, fieldVisibility: FieldVisibility, enableDecimalLogicalType: Boolean) {
val protocol = Protocol.parse(avpr)
val compiler = new SpecificCompiler(protocol)
compiler.setStringType(stringType)
compiler.setFieldVisibility(fieldVisibility)
compiler.setEnableDecimalLogicalType(enableDecimalLogicalType)
compiler.compileToDestination(null, target)
def compileAvprs(avprs: Seq[File], target: File, log: Logger, stringType: StringType, fieldVisibility: FieldVisibility, enableDecimalLogicalType: Boolean) = {
avprs.foreach { avpr =>
log.info(s"Compiling Avro protocol $avpr")
val protocol = Protocol.parse(avpr)
val compiler = new SpecificCompiler(protocol)
compiler.setStringType(stringType)
compiler.setFieldVisibility(fieldVisibility)
compiler.setEnableDecimalLogicalType(enableDecimalLogicalType)
compiler.compileToDestination(null, target)
}
}

private[this] def compileAvroSchema(srcDir: File,
private[this] def compileAvroSchema(srcDirs: Seq[File],
target: File,
includes: Seq[File],
log: Logger,
stringType: StringType,
fieldVisibility: FieldVisibility,
enableDecimalLogicalType: Boolean,
useNamespace: Boolean,
builder: SchemaParserBuilder): Set[File] = {
(srcDir ** AvroAvdlFilter).get.foreach { idl =>
log.info(s"Compiling Avro IDL $idl")
compileIdl(idl, target, stringType, fieldVisibility, enableDecimalLogicalType)
}

val includeAvscs = includes.flatMap { include =>
val includes = (include ** AvroAvscFilter).get
includes.foreach(s => log.info(s"Including: $s"))
includes.map(avsc => new AvroFileRef(include, avsc.relativeTo(include).get.toString))
}

val avscs = (srcDir ** AvroAvscFilter).get.map { avsc =>
log.info(s"Compiling Avro schemas $avsc")
new AvroFileRef(srcDir, avsc.relativeTo(srcDir).get.toString)
}
val avdls = srcDirs.flatMap(d => (d ** AvroAvdlFilter).get)
val avscs = srcDirs.flatMap(d => (d ** AvroAvscFilter).get.map(avsc => new AvroFileRef(d, avsc.relativeTo(d).get.toString)))
val avprs = srcDirs.flatMap(d => (d ** AvroAvrpFilter).get)

compileAvscs(includeAvscs ++ avscs, target, stringType, fieldVisibility, enableDecimalLogicalType, useNamespace, builder)

(srcDir ** AvroAvrpFilter).get.foreach { avpr =>
log.info(s"Compiling Avro protocol $avpr")
compileAvpr(avpr, target, stringType, fieldVisibility, enableDecimalLogicalType)
}
compileIdls(avdls, target, log, stringType, fieldVisibility, enableDecimalLogicalType)
compileAvscs(avscs, target, log, stringType, fieldVisibility, enableDecimalLogicalType, useNamespace, builder)
compileAvprs(avprs, target, log, stringType, fieldVisibility, enableDecimalLogicalType)

(target ** JavaFileFilter).get.toSet
}

private def sourceGeneratorTask(key: TaskKey[Seq[File]]) = Def.task {
val out = (key / streams).value
val externalSrcDir = (avroUnpackDependencies / target).value
val srcDir = avroSource.value
val externalSrcDir = (avroUnpackDependencies / target).value
val includes = avroIncludes.value
val srcDirs = Seq(externalSrcDir, srcDir) ++ includes
val outDir = (key / target).value
val strType = StringType.valueOf(avroStringType.value)
val fieldVis = SpecificCompiler.FieldVisibility.valueOf(avroFieldVisibility.value.toUpperCase)
Expand All @@ -218,13 +212,11 @@ object SbtAvro extends AutoPlugin {
val cachedCompile = {
FileFunction.cached(out.cacheDirectory / "avro", FilesInfo.lastModified, FilesInfo.exists) { _ =>
out.log.info(s"Avro compiler using stringType=$strType")
compileAvroSchema(externalSrcDir, outDir, Seq.empty, out.log, strType, fieldVis, enbDecimal, useNs, builder)
compileAvroSchema(srcDir, outDir, includes, out.log, strType, fieldVis, enbDecimal, useNs, builder)

compileAvroSchema(srcDirs, outDir, out.log, strType, fieldVis, enbDecimal, useNs, builder)
}
}

cachedCompile(((externalSrcDir +++ srcDir) ** AvroFilter).get.toSet).toSeq
cachedCompile((srcDirs ** AvroFilter).get.toSet).toSeq
}

}
3 changes: 1 addition & 2 deletions src/sbt-test/sbt-avro/publishing/build.sbt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,7 @@ lazy val `transitive`: Project = project
Compile / packageAvro / publishArtifact := true,
libraryDependencies ++= Seq(
"com.cavorite" % "external" % "0.0.1-SNAPSHOT" classifier "avro",
),
avroIncludes := Seq((avroUnpackDependencies / target).value)
)
)

lazy val root: Project = project
Expand Down
4 changes: 3 additions & 1 deletion src/test/scala/sbtavro/SbtAvroSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import com.spotify.avro.mojo.AvroFileRef
import org.apache.avro.compiler.specific.SpecificCompiler.FieldVisibility
import org.apache.avro.generic.GenericData.StringType
import org.specs2.mutable.Specification
import sbt.util.Logger

/**
* Created by jeromewacongne on 06/08/2015.
Expand All @@ -14,6 +15,7 @@ class SbtAvroSpec extends Specification {
val builder = DefaultSchemaParserBuilder.default()
val sourceDir = new File(getClass.getClassLoader.getResource("avro").toURI)
val targetDir = new File(sourceDir.getParentFile, "generated")
val logger = Logger.Null

val fullyQualifiedNames = Seq(
new File(sourceDir, "a.avsc"),
Expand Down Expand Up @@ -73,7 +75,7 @@ class SbtAvroSpec extends Specification {
_eJavaFile.delete()

val refs = sourceFiles.map(s => new AvroFileRef(sourceDir, s.getName))
SbtAvro.compileAvscs(refs, targetDir, StringType.CharSequence, FieldVisibility.PUBLIC_DEPRECATED, true, false, builder)
SbtAvro.compileAvscs(refs, targetDir, logger, StringType.CharSequence, FieldVisibility.PUBLIC_DEPRECATED, true, false, builder)

aJavaFile.isFile must beTrue
bJavaFile.isFile must beTrue
Expand Down

0 comments on commit b8b2cb9

Please sign in to comment.