From 0027adb77b42f2d6317ceb0ca69b2534d5ba15be Mon Sep 17 00:00:00 2001 From: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Date: Tue, 20 Dec 2022 09:40:44 -0800 Subject: [PATCH 1/5] Fix failing test --- ...ltBigQueryDenormalizedRecordFormatter.java | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryDenormalizedRecordFormatter.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryDenormalizedRecordFormatter.java index 286d65e114863..45e2d51e9ca73 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryDenormalizedRecordFormatter.java +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryDenormalizedRecordFormatter.java @@ -40,6 +40,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,7 +69,7 @@ private ArrayFormatter getArrayFormatter() { return arrayFormatter; } - public void setArrayFormatter(ArrayFormatter arrayFormatter) { + public void setArrayFormatter(final ArrayFormatter arrayFormatter) { this.arrayFormatter = arrayFormatter; this.jsonSchema = formatJsonSchema(this.originalJsonSchema.deepCopy()); this.bigQuerySchema = getBigQuerySchema(jsonSchema); @@ -76,7 +77,7 @@ public void setArrayFormatter(ArrayFormatter arrayFormatter) { @Override protected JsonNode formatJsonSchema(final JsonNode jsonSchema) { - var modifiedJsonSchema = jsonSchema.deepCopy(); // Issue #5912 is reopened (PR #11166) formatAllOfAndAnyOfFields(namingResolver, jsonSchema); + final var modifiedJsonSchema = jsonSchema.deepCopy(); // Issue #5912 is reopened (PR #11166) formatAllOfAndAnyOfFields(namingResolver, jsonSchema); getArrayFormatter().populateEmptyArrays(modifiedJsonSchema); getArrayFormatter().surroundArraysByObjects(modifiedJsonSchema); return modifiedJsonSchema; @@ -117,7 +118,7 @@ private JsonNode formatData(final FieldList fields, final JsonNode root) { if (fields == null) { return root; } - JsonNode formattedData; + final JsonNode formattedData; if (root.isObject()) { formattedData = getObjectNode(fields, root); } else if (root.isArray()) { @@ -151,7 +152,7 @@ private JsonNode getArrayNode(final FieldList fields, final JsonNode root) { } else { subFields = arrayField.getSubFields(); } - List arrayItems = MoreIterators.toList(root.elements()).stream() + final List arrayItems = MoreIterators.toList(root.elements()).stream() .map(p -> formatData(subFields, p)) .toList(); @@ -245,15 +246,15 @@ private static JsonNode getFileDefinition(final JsonNode fieldDefinition) { } private static JsonNode allOfAndAnyOfFieldProcessing(final String fieldName, final JsonNode fieldDefinition) { - ObjectReader reader = mapper.readerFor(new TypeReference>() {}); - List list; + final ObjectReader reader = mapper.readerFor(new TypeReference>() {}); + final List list; try { list = reader.readValue(fieldDefinition.get(fieldName)); - } catch (IOException e) { + } catch (final IOException e) { throw new IllegalStateException( String.format("Failed to read and process the following field - %s", fieldDefinition)); } - ObjectNode objectNode = mapper.createObjectNode(); + final ObjectNode objectNode = mapper.createObjectNode(); list.forEach(field -> { objectNode.set("big_query_" + field.get("type").asText(), field); }); @@ -268,8 +269,8 @@ private static JsonNode allOfAndAnyOfFieldProcessing(final String fieldName, fin private static Builder getField(final StandardNameTransformer namingResolver, final String key, final JsonNode fieldDefinition) { final String fieldName = namingResolver.getIdentifier(key); final Builder builder = Field.newBuilder(fieldName, StandardSQLTypeName.STRING); - JsonNode updatedFileDefinition = getFileDefinition(fieldDefinition); - JsonNode type = updatedFileDefinition.get(TYPE_FIELD); + final JsonNode updatedFileDefinition = getFileDefinition(fieldDefinition); + final JsonNode type = updatedFileDefinition.get(TYPE_FIELD); final JsonNode airbyteType = updatedFileDefinition.get(AIRBYTE_TYPE); final List fieldTypes = getTypes(fieldName, type); for (int i = 0; i < fieldTypes.size(); i++) { @@ -288,9 +289,11 @@ private static Builder getField(final StandardNameTransformer namingResolver, fi builder.setType(primaryType.getBigQueryType()); } case NUMBER -> { - if (airbyteType != null && airbyteType.asText().equals("big_integer")) { + if (airbyteType != null + && StringUtils.equalsAnyIgnoreCase(airbyteType.asText(), + "big_integer", "integer")) { builder.setType(StandardSQLTypeName.INT64); - } else { + } else { builder.setType(primaryType.getBigQueryType()); } } From d9764a5be594e4c453a4e256fa525b8598f46692 Mon Sep 17 00:00:00 2001 From: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Date: Tue, 20 Dec 2022 22:00:36 -0800 Subject: [PATCH 2/5] Bigquery-denormalized update spec plus destination_definitions.yaml to fix integration tests --- .../src/main/resources/seed/destination_definitions.yaml | 4 ---- .../src/main/resources/spec.json | 2 +- .../BigQueryDenormalizedDestinationAcceptanceTest.java | 5 ----- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 81a45c0826177..f03606ae76a80 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -60,10 +60,6 @@ dockerImageTag: 1.2.9 documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery icon: bigquery.svg - normalizationConfig: - normalizationRepository: airbyte/normalization - normalizationTag: 0.2.25 - normalizationIntegrationType: bigquery resourceRequirements: jobSpecific: - jobType: sync diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/resources/spec.json index fa11d9c3745df..9727c9b323cb6 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/resources/spec.json @@ -2,7 +2,7 @@ "documentationUrl": "https://docs.airbyte.com/integrations/destinations/bigquery", "supportsIncremental": true, "supportsNormalization": false, - "supportsDBT": true, + "supportsDBT": false, "supported_destination_sync_modes": ["overwrite", "append"], "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestinationAcceptanceTest.java index 2efc146da794e..378e0312fe664 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestinationAcceptanceTest.java @@ -81,11 +81,6 @@ protected JsonNode getFailCheckConfig() { return config; } - @Override - protected boolean supportsDBT() { - return true; - } - @Override protected boolean implementsNamespaces() { return true; From e49117aaf2aacabddd7c88a2891f3384f72bc3ef Mon Sep 17 00:00:00 2001 From: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Date: Wed, 28 Dec 2022 11:45:54 -0800 Subject: [PATCH 3/5] sanity --- .../formatter/DefaultBigQueryDenormalizedRecordFormatter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryDenormalizedRecordFormatter.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryDenormalizedRecordFormatter.java index 45e2d51e9ca73..59b86f36a5f35 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryDenormalizedRecordFormatter.java +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryDenormalizedRecordFormatter.java @@ -293,7 +293,7 @@ private static Builder getField(final StandardNameTransformer namingResolver, fi && StringUtils.equalsAnyIgnoreCase(airbyteType.asText(), "big_integer", "integer")) { builder.setType(StandardSQLTypeName.INT64); - } else { + } else { builder.setType(primaryType.getBigQueryType()); } } From 6714026974e4ffaa87a3cfbb00e1fbbf0187cc1e Mon Sep 17 00:00:00 2001 From: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Date: Wed, 4 Jan 2023 11:59:58 +0530 Subject: [PATCH 4/5] Update version number and release note --- .../connectors/destination-bigquery-denormalized/Dockerfile | 2 +- docs/integrations/destinations/bigquery.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile index 53486620dae2f..1bdb65e603efc 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.2.9 +LABEL io.airbyte.version=1.2.10 LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index ccb7b39381747..449767925c63f 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -98,7 +98,7 @@ Airbyte converts any invalid characters into `_` characters when writing data. H |:------------------------------------|:--------------|:---------------------------| | DATE | DATE | DATE | | STRING (BASE64) | STRING | STRING | -| NUMBER | FLOAT | FLOAT | +| NUMBER | FLOAT | NUMBER | | OBJECT | STRING | RECORD | | STRING | STRING | STRING | | BOOLEAN | BOOLEAN | BOOLEAN | @@ -191,6 +191,7 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------| +| 1.2.10 | 2023-01-04 | [#20730](https://github.com/airbytehq/airbyte/pull/20730) | An incoming source Number type will create a big query integer rather than a float. | | 1.2.9 | 2022-12-14 | [#20501](https://github.com/airbytehq/airbyte/pull/20501) | Report GCS staging failures that occur during connection check | | 1.2.8 | 2022-11-22 | [#19489](https://github.com/airbytehq/airbyte/pull/19489) | Added non-billable projects handle to check connection stage | | 1.2.7 | 2022-11-11 | [#19358](https://github.com/airbytehq/airbyte/pull/19358) | Fixed check method to capture mismatch dataset location | From 1d10eb2654f61aaccce5992b0989755f39658374 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Wed, 4 Jan 2023 07:17:27 +0000 Subject: [PATCH 5/5] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 71b4bfc5a0c12..0fc80eab19aa1 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -57,7 +57,7 @@ - name: BigQuery (denormalized typed struct) destinationDefinitionId: 079d5540-f236-4294-ba7c-ade8fd918496 dockerRepository: airbyte/destination-bigquery-denormalized - dockerImageTag: 1.2.9 + dockerImageTag: 1.2.10 documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery icon: bigquery.svg resourceRequirements: diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index a4502ff28591f..e360fe769b47d 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -831,7 +831,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/destination-bigquery-denormalized:1.2.9" +- dockerImage: "airbyte/destination-bigquery-denormalized:1.2.10" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/bigquery" connectionSpecification: @@ -1025,7 +1025,7 @@ order: 5 supportsIncremental: true supportsNormalization: false - supportsDBT: true + supportsDBT: false supported_destination_sync_modes: - "overwrite" - "append"