Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Avoid unpack conflict #217

Merged
merged 1 commit into from
Dec 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ to generate the avro classes.
| `avroFieldVisibility` | `public` | Field visibility for the properties. Possible values: `private`, `public`. |
| `avroOptionalGetters` | `false` (requires avro `1.10+`) | Generate getters that return `Optional` for nullable fields. |
| `avroStringType` | `CharSequence` | Type for representing strings. Possible values: `CharSequence`, `String`, `Utf8`. |
| `avroUseNamespace` | `false` | Validate that directory layout reflects namespaces, i.e. `com/myorg/MyRecord.avsc`. |
| `avroVersion` | `1.12.0` | Avro version to use in the project. |

### Scoped settings (Compile/Test)
Expand Down
3 changes: 1 addition & 2 deletions api/src/main/java/com/github/sbt/avro/AvroCompiler.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,12 @@ public interface AvroCompiler {

void setStringType(String stringType);
void setFieldVisibility(String fieldVisibility);
void setUseNamespace(boolean useNamespace);
void setEnableDecimalLogicalType(boolean enableDecimalLogicalType);
void setCreateSetters(boolean createSetters);
void setOptionalGetters(boolean optionalGetters);

void recompile(Class<?>[] records, File target) throws Exception;
void compileIdls(File[] idls, File target) throws Exception;
void compileAvscs(AvroFileRef[] avscs, File target) throws Exception;
void compileAvscs(File[] avscs, File target) throws Exception;
void compileAvprs(File[] avprs, File target) throws Exception;
}
59 changes: 0 additions & 59 deletions api/src/main/java/com/github/sbt/avro/AvroFileRef.java

This file was deleted.

47 changes: 8 additions & 39 deletions bridge/src/main/java/com/github/sbt/avro/AvroCompilerBridge.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ public class AvroCompilerBridge implements AvroCompiler {

protected StringType stringType;
protected FieldVisibility fieldVisibility;
protected boolean useNamespace;
protected boolean enableDecimalLogicalType;
protected boolean createSetters;
protected boolean optionalGetters;
Expand All @@ -41,11 +40,6 @@ public void setFieldVisibility(String fieldVisibility) {
this.fieldVisibility = FieldVisibility.valueOf(fieldVisibility);
}

@Override
public void setUseNamespace(boolean useNamespace) {
this.useNamespace = useNamespace;
}

@Override
public void setEnableDecimalLogicalType(boolean enableDecimalLogicalType) {
this.enableDecimalLogicalType = enableDecimalLogicalType;
Expand Down Expand Up @@ -101,21 +95,16 @@ public void compileIdls(File[] idls, File target) throws Exception {
}

@Override
public void compileAvscs(AvroFileRef[] avscs, File target) throws Exception {
List<AvroFileRef> files = new ArrayList<>(avscs.length);
for (AvroFileRef ref : avscs) {
System.out.println("Compiling Avro schema: " + ref.getFile());
files.add(ref);
}
Map<AvroFileRef, Schema> schemas = parser.parseFiles(files);
if (useNamespace) {
for (Map.Entry<AvroFileRef, Schema> s: schemas.entrySet()) {
validateParsedSchema(s.getKey(), s.getValue());
}
public void compileAvscs(File[] avscs, File target) throws Exception {
List<File> files = new ArrayList<>(avscs.length);
for (File schema : avscs) {
System.out.println("Compiling Avro schema: " + schema);
files.add(schema);
}
Map<File, Schema> schemas = parser.parseFiles(files);

for (Map.Entry<AvroFileRef, Schema> entry: schemas.entrySet()) {
File file = entry.getKey().getFile();
for (Map.Entry<File, Schema> entry: schemas.entrySet()) {
File file = entry.getKey();
Schema schema = entry.getValue();
SpecificCompiler compiler = new SpecificCompiler(schema);
configureCompiler(compiler);
Expand All @@ -133,24 +122,4 @@ public void compileAvprs(File[] avprs, File target) throws Exception {
compiler.compileToDestination(null, target);
}
}

private void validateParsedSchema(AvroFileRef src, Schema schema) {
if (useNamespace) {
if (schema.getType() != Schema.Type.RECORD && schema.getType() != Schema.Type.ENUM) {
throw new SchemaGenerationException(String.format(
"Error compiling schema file %s. "
+ "Only one root RECORD or ENUM type is allowed per file.",
src
));
} else if (!src.pathToClassName().equals(schema.getFullName())) {
throw new SchemaGenerationException(String.format(
"Error compiling schema file %s. "
+ "File class name %s does not match record class name %s",
src,
src.pathToClassName(),
schema.getFullName()
));
}
}
}
}
15 changes: 8 additions & 7 deletions bridge/src/main/java/com/github/sbt/avro/AvscFilesParser.java
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import org.apache.avro.Schema;

import java.io.IOException;
import java.io.File;
import java.util.*;
import java.util.function.Supplier;
import java.util.stream.Collectors;
Expand All @@ -30,20 +31,20 @@ public void addTypes(Iterable<Schema> types) {
}
}

public Map<AvroFileRef, Schema> parseFiles(Collection<AvroFileRef> files) {
Set<AvroFileRef> unparsedFiles = new HashSet<>(files);
Map<AvroFileRef, Schema> parsedFiles = new HashMap<>();
Map<AvroFileRef, Exception> parseExceptions = new HashMap<>();
public Map<File, Schema> parseFiles(Collection<File> files) {
Set<File> unparsedFiles = new HashSet<>(files);
Map<File, Schema> parsedFiles = new HashMap<>();
Map<File, Exception> parseExceptions = new HashMap<>();

Schema.Parser parser = unstashParser();
boolean progressed = true;
while (progressed && !unparsedFiles.isEmpty()) {
progressed = false;
parseExceptions.clear();

for (AvroFileRef file : unparsedFiles) {
for (File file : unparsedFiles) {
try {
Schema schema = parser.parse(file.getFile());
Schema schema = parser.parse(file);
parsedFiles.put(file, schema);
progressed = true;
stashParser(parser);
Expand All @@ -64,7 +65,7 @@ public Map<AvroFileRef, Schema> parseFiles(Collection<AvroFileRef> files) {
String message = Optional.ofNullable(parseExceptions.get(f))
.map(Exception::getMessage)
.orElse("Unknown error");
return f.getFile().getName() + ": " + message;
return f.getName() + ": " + message;
})
.collect(Collectors.joining(",\n"));

Expand Down
22 changes: 11 additions & 11 deletions bridge/src/test/scala/com/github/sbt/avro/AvscFilesParserSpec.scala
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,19 @@ class AvscFilesParserSpec extends Specification {
"It should be possible to compile types depending on others if source files are provided in right order" >> {
val parser = new AvscFilesParser()
val fullyQualifiedNames = Seq(
new AvroFileRef(sourceDir, "a.avsc"),
new AvroFileRef(sourceDir, "b.avsc"),
new AvroFileRef(sourceDir, "c.avsc"),
new AvroFileRef(sourceDir, "d.avsc"),
new AvroFileRef(sourceDir, "e.avsc")
new File(sourceDir, "a.avsc"),
new File(sourceDir, "b.avsc"),
new File(sourceDir, "c.avsc"),
new File(sourceDir, "d.avsc"),
new File(sourceDir, "e.avsc")
)

val simpleNames = Seq(
new AvroFileRef(sourceDir, "_a.avsc"),
new AvroFileRef(sourceDir, "_b.avsc"),
new AvroFileRef(sourceDir, "_c.avsc"),
new AvroFileRef(sourceDir, "_d.avsc"),
new AvroFileRef(sourceDir, "_e.avsc")
new File(sourceDir, "_a.avsc"),
new File(sourceDir, "_b.avsc"),
new File(sourceDir, "_c.avsc"),
new File(sourceDir, "_d.avsc"),
new File(sourceDir, "_e.avsc")
)

val sourceFiles = fullyQualifiedNames ++ simpleNames
Expand Down Expand Up @@ -66,7 +66,7 @@ class AvscFilesParserSpec extends Specification {
| ]
|}""".stripMargin
)
val parent = new AvroFileRef(sourceDir, "test_records.avsc")
val parent = new File(sourceDir, "test_records.avsc")
parser.addTypes(Seq(dependant).asJava)
val schemas = parser.parseFiles(Seq(parent).asJava)
val names = schemas.asScala.values.map(_.getFullName)
Expand Down
22 changes: 11 additions & 11 deletions plugin/src/main/scala/com/github/sbt/avro/SbtAvro.scala
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ object SbtAvro extends AutoPlugin {
val avroSource = settingKey[File]("Default Avro source directory for *.avsc, *.avdl and *.avpr files.")
val avroStringType = settingKey[String]("Type for representing strings. Possible values: CharSequence, String, Utf8.")
val avroUnmanagedSourceDirectories = settingKey[Seq[File]]("Unmanaged Avro source directories, which contain manually created sources.")
val avroUseNamespace = settingKey[Boolean]("Validate that directory layout reflects namespaces, i.e. com/myorg/MyRecord.avsc.")
val avroVersion = settingKey[String]("Avro version to use in the project.")

val avroGenerate = taskKey[Seq[File]]("Generate Java sources for Avro schemas.")
Expand All @@ -60,7 +59,6 @@ object SbtAvro extends AutoPlugin {
avroFieldVisibility := "public",
avroOptionalGetters := false,
avroStringType := "CharSequence",
avroUseNamespace := false,

// addArtifact doesn't take publishArtifact setting in account
artifacts ++= Classpaths.artifactDefs(avroArtifactTasks).value,
Expand Down Expand Up @@ -142,11 +140,13 @@ object SbtAvro extends AutoPlugin {
inStyle = FilesInfo.lastModified,
outStyle = FilesInfo.exists
) { deps =>
IO.createDirectory(extractTarget)
// dedicated directory per artifact to avoid name conflicts
val depTarget = extractTarget / jar.base
IO.createDirectory(depTarget)
deps.flatMap { dep =>
val filter = includeFilter -- excludeFilter
val (avroSpecs, filtered) = IO
.unzip(dep, extractTarget, AvroFilter)
.unzip(dep, depTarget, AvroFilter)
.partition(filter.accept)
IO.delete(filtered)
if (avroSpecs.nonEmpty) {
Expand Down Expand Up @@ -178,14 +178,19 @@ object SbtAvro extends AutoPlugin {
.toSeq
.map { case (_, _, _, f) => f }

unpack(
val unpacked = unpack(
cacheBaseDirectory = cacheBaseDirectory,
deps = avroArtifacts,
extractTarget = (key / target).value,
includeFilter = (key / includeFilter).value,
excludeFilter = (key / excludeFilter).value,
streams = (key / streams).value
)

val previouslyUnpacked = key.previous.toSeq.flatten
IO.delete(previouslyUnpacked.diff(unpacked))

unpacked
}

private def sourceGeneratorTask(key: TaskKey[Seq[File]]) = Def.task {
Expand Down Expand Up @@ -239,18 +244,13 @@ object SbtAvro extends AutoPlugin {

compiler.setStringType(avroStringType.value)
compiler.setFieldVisibility(avroFieldVisibility.value.toUpperCase)
compiler.setUseNamespace(avroUseNamespace.value)
compiler.setEnableDecimalLogicalType(avroEnableDecimalLogicalType.value)
compiler.setCreateSetters(avroCreateSetters.value)
compiler.setOptionalGetters(avroOptionalGetters.value)

val recs = records.map(avroClassLoader.loadClass)
val avdls = srcDirs.flatMap(d => (d ** AvroAvdlFilter).get())
val avscs = srcDirs.flatMap(d =>
(d ** AvroAvscFilter)
.get()
.map(avsc => new AvroFileRef(d, avsc.relativeTo(d).get.toString))
)
val avscs = srcDirs.flatMap(d => (d ** AvroAvscFilter).get())
val avprs = srcDirs.flatMap(d => (d ** AvroAvrpFilter).get())

out.log.info(
Expand Down
12 changes: 6 additions & 6 deletions plugin/src/sbt-test/sbt-avro/publishing/build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -69,11 +69,11 @@ lazy val root: Project = project
Compile / avroUnpackDependencies / excludeFilter := (Compile / avroUnpackDependencies / excludeFilter).value || "exclude.avsc",

Compile / checkUnpacked := {
exists(crossTarget.value / "src_managed" / "avro" / "main" / "com" / "github" / "sbt" / "avro" / "test" / "external" / "avdl.avdl")
exists(crossTarget.value / "src_managed" / "avro" / "main" / "com" / "github" / "sbt" / "avro" / "test" / "external" / "avpr.avpr")
exists(crossTarget.value / "src_managed" / "avro" / "main" / "com" / "github" / "sbt" / "avro" / "test" / "external" / "avsc.avsc")
absent(crossTarget.value / "src_managed" / "avro" / "main" / "com" / "github" / "sbt" / "avro" / "test" / "external" / "exclude.avsc")
exists(crossTarget.value / "src_managed" / "avro" / "main" / "com" / "github" / "sbt" / "avro" / "test" / "transitive" / "avsc.avsc")
exists(crossTarget.value / "src_managed" / "avro" / "main" / "external-avro" / "avdl.avdl")
exists(crossTarget.value / "src_managed" / "avro" / "main" / "external-avro" / "avpr.avpr")
exists(crossTarget.value / "src_managed" / "avro" / "main" / "external-avro" / "avsc.avsc")
absent(crossTarget.value / "src_managed" / "avro" / "main" / "external-avro" / "exclude.avsc")
exists(crossTarget.value / "src_managed" / "avro" / "main" / "transitive-avro" / "avsc.avsc")
},
Compile / checkGenerated := {
exists(crossTarget.value / "src_managed" / "compiled_avro" / "main" / "com" / "github" / "sbt" / "avro" / "test" / "external" / "Avdl.java")
Expand All @@ -82,7 +82,7 @@ lazy val root: Project = project
exists(crossTarget.value / "src_managed" / "compiled_avro" / "main" / "com" / "github" / "sbt" / "avro" / "test" / "transitive" / "Avsc.java")
},
Test / checkUnpacked := {
exists(crossTarget.value / "src_managed" / "avro" / "test" / "test.avsc")
exists(crossTarget.value / "src_managed" / "avro" / "test" / "transitive-tests" / "test.avsc")
},
Test / checkGenerated := {
exists(crossTarget.value / "src_managed" / "compiled_avro" / "test" / "com" / "github" / "sbt" / "avro" / "test" / "transitive" / "Test.java")
Expand Down