Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Destination S3-V2: avro schema matches legacy CDK #51553

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+65 −2
Open

Destination S3-V2: avro schema matches legacy CDK #51553

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions ...olkits/load-avro/src/main/kotlin/io/airbyte/cdk/load/data/avro/AirbyteTypeToAvroSchema.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import io.airbyte.cdk.load.data.TimestampTypeWithoutTimezone
import io.airbyte.cdk.load.data.Transformations
import io.airbyte.cdk.load.data.UnionType
import io.airbyte.cdk.load.data.UnknownType
import io.airbyte.cdk.load.message.Meta
import org.apache.avro.LogicalTypes
import org.apache.avro.Schema
import org.apache.avro.SchemaBuilder
Expand All @@ -40,8 +41,19 @@ class AirbyteTypeToAvroSchema {
SchemaBuilder.record(recordName).namespace(namespaceMangled).fields()
airbyteSchema.properties.entries
.fold(builder) { acc, (name, field) ->
val converted = convert(field.type, path + name)
val propertySchema = maybeMakeNullable(field, converted)
val propertySchema =
when (name) {
Meta.COLUMN_NAME_AB_EXTRACTED_AT ->
LogicalTypes.timestampMillis()
.addToSchema(Schema.create(Schema.Type.LONG))
Meta.COLUMN_NAME_AB_RAW_ID ->
LogicalTypes.uuid()
.addToSchema(Schema.create(Schema.Type.STRING))
else -> {
val converted = convert(field.type, path + name)
maybeMakeNullable(field, converted)
}
}
val nameMangled = Transformations.toAvroSafeName(name)
acc.name(nameMangled).type(propertySchema).let {
if (field.nullable) {
Expand Down
51 changes: 51 additions & 0 deletions airbyte-cdk/bulk/toolkits/load-avro/src/test/kotlin/AirbyteTypeToAvroSchemaTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@ import io.airbyte.cdk.load.data.FieldType
import io.airbyte.cdk.load.data.ObjectType
import io.airbyte.cdk.load.data.StringType
import io.airbyte.cdk.load.data.avro.toAvroSchema
import io.airbyte.cdk.load.data.withAirbyteMeta
import io.airbyte.cdk.load.message.Meta
import org.apache.avro.LogicalTypes
import org.apache.avro.Schema
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertDoesNotThrow

Expand All @@ -23,4 +28,50 @@ class AirbyteTypeToAvroSchemaTest {
val descriptor = DestinationStream.Descriptor("test", "stream")
assertDoesNotThrow { schema.toAvroSchema(descriptor) }
}

@Test
fun `test airbyte meta schema`() {
val schema = ObjectType(linkedMapOf()).withAirbyteMeta()
val descriptor = DestinationStream.Descriptor("test", "stream")
val avroSchema = schema.toAvroSchema(descriptor)

assertEquals(
avroSchema.getField(Meta.COLUMN_NAME_AB_RAW_ID).schema().type,
Schema.Type.STRING
)
assertEquals(
avroSchema.getField(Meta.COLUMN_NAME_AB_RAW_ID).schema().logicalType,
LogicalTypes.uuid()
)
assertEquals(
avroSchema.getField(Meta.COLUMN_NAME_AB_EXTRACTED_AT).schema().type,
Schema.Type.LONG
)
assertEquals(
avroSchema.getField(Meta.COLUMN_NAME_AB_EXTRACTED_AT).schema().logicalType,
LogicalTypes.timestampMillis()
)
assertEquals(
avroSchema.getField(Meta.COLUMN_NAME_AB_GENERATION_ID).schema().type,
Schema.Type.LONG
)

val metaSchema = avroSchema.getField(Meta.COLUMN_NAME_AB_META).schema()
assertEquals(metaSchema.type, Schema.Type.RECORD)
assertEquals(metaSchema.fields.size, 2)

// Meta field 1
val changesSchema = metaSchema.getField("changes").schema()
assertEquals(changesSchema.type, Schema.Type.ARRAY)

val changeSchema = changesSchema.elementType
assertEquals(changeSchema.type, Schema.Type.RECORD)
assertEquals(changeSchema.fields.size, 3)
assertEquals(changeSchema.getField("field").schema().type, Schema.Type.STRING)
assertEquals(changeSchema.getField("change").schema().type, Schema.Type.STRING)
assertEquals(changeSchema.getField("reason").schema().type, Schema.Type.STRING)

// Meta field 2
assertEquals(metaSchema.getField("sync_id").schema().type, Schema.Type.LONG)
}
}
Loading