From a69e86ba884888d53ab30da8e10021b4a18cfdac Mon Sep 17 00:00:00 2001 From: Prashant Mishra <11733935+prash-mi@users.noreply.github.com> Date: Fri, 14 Oct 2022 21:31:16 +0530 Subject: [PATCH] chore: Documentation of writeJsonStream methods (#1290) --- docs/src/main/asciidoc/bigquery.adoc | 57 +++++++++++++++++++++++ docs/src/main/md/bigquery.md | 68 +++++++++++++++++++++++++--- 2 files changed, 118 insertions(+), 7 deletions(-) diff --git a/docs/src/main/asciidoc/bigquery.adoc b/docs/src/main/asciidoc/bigquery.adoc index d36505bb00..cccc3e5479 100644 --- a/docs/src/main/asciidoc/bigquery.adoc +++ b/docs/src/main/asciidoc/bigquery.adoc @@ -36,6 +36,8 @@ The following application properties may be configured with Spring Cloud GCP Big | `spring.cloud.gcp.bigquery.enabled` | Enables or disables Spring Cloud GCP BigQuery autoconfiguration. | No | `true` | `spring.cloud.gcp.bigquery.project-id` | GCP project ID of the project using BigQuery APIs, if different from the one in the <>. | No | Project ID is typically inferred from https://cloud.google.com/sdk/gcloud/reference/config/set[`gcloud`] configuration. | `spring.cloud.gcp.bigquery.credentials.location` | Credentials file location for authenticating with the Google Cloud BigQuery APIs, if different from the ones in the <> | No | Inferred from https://cloud.google.com/docs/authentication/production[Application Default Credentials], typically set by https://cloud.google.com/sdk/gcloud/reference/auth/application-default[`gcloud`]. +| `spring.cloud.gcp.bigquery.jsonWriterBatchSize` | Batch size which will be used by `BigQueryJsonDataWriter` while using https://cloud.google.com/bigquery/docs/write-api[BigQuery Storage Write API]. Note too large or too low values might impact performance. | No | 1000 +| `spring.cloud.gcp.bigquery.threadPoolSize` | The size of thread pool of `ThreadPoolTaskScheduler` which is used by `BigQueryTemplate` | No | 4 |=========================================================================== ==== BigQuery Client Object @@ -93,6 +95,61 @@ public void loadData(InputStream dataInputStream, String tableName) { } ---- +Below is a code snippet of how to load a https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-json[newline-delimited JSON] data `InputStream` to a BigQuery table. This implementation uses the https://cloud.google.com/bigquery/docs/write-api[BigQuery Storage Write API]. +https://github.com/GoogleCloudPlatform/spring-cloud-gcp/tree/main/spring-cloud-gcp-bigquery/src/test/resources/data.json[Here] is a sample newline-delimited JSON file which can be used for testing this functionality. + +[source,java] +---- +// BigQuery client object provided by our autoconfiguration. +@Autowired +BigQueryTemplate bigQueryTemplate; + + /** + * This method loads the InputStream of the newline-delimited JSON records to be written in the given table. + * @param tableName name of the table where the data is expected to be written + * @param jsonInputStream InputStream of the newline-delimited JSON records to be written in the given table + */ + public void loadJsonStream(String tableName, InputStream jsonInputStream) + throws ExecutionException, InterruptedException { + ListenableFuture writeApFuture = + bigQueryTemplate.writeJsonStream(tableName, jsonInputStream); + WriteApiResponse apiRes = writeApFuture.get();//get the WriteApiResponse + if (!apiRes.isSuccessful()){ + List errors = apiRes.getErrors(); + //TODO(developer): process the List of StorageError + } + //else the write process has been successful + } +---- + +Below is a code snippet of how to create table and then load a https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-json[newline-delimited JSON] data `InputStream` to a BigQuery table. This implementation uses the https://cloud.google.com/bigquery/docs/write-api[BigQuery Storage Write API]. +https://github.com/GoogleCloudPlatform/spring-cloud-gcp/tree/main/spring-cloud-gcp-bigquery/src/test/resources/data.json[Here] is a sample newline-delimited JSON file which can be used for testing this functionality. + +[source,java] +---- +// BigQuery client object provided by our autoconfiguration. +@Autowired +BigQueryTemplate bigQueryTemplate; + + /** + * This method created a table with the given name and schema and then loads the InputStream of the newline-delimited JSON records in it. + * @param tableName name of the table where the data is expected to be written + * @param jsonInputStream InputStream of the newline-delimited JSON records to be written in the given table + * @param tableSchema Schema of the table which is required to be created + */ + public void createTableAndloadJsonStream(String tableName, InputStream jsonInputStream, Schema tableSchema) + throws ExecutionException, InterruptedException { + ListenableFuture writeApFuture = + bigQueryTemplate.writeJsonStream(tableName, jsonInputStream, tableSchema);//using the overloaded method which created the table when tableSchema is passed + WriteApiResponse apiRes = writeApFuture.get();//get the WriteApiResponse + if (!apiRes.isSuccessful()){ + List errors = apiRes.getErrors(); + //TODO(developer): process the List of StorageError + } + //else the write process has been successful + } +---- + === Spring Integration Spring Cloud GCP BigQuery also provides a Spring Integration message handler `BigQueryFileMessageHandler`. diff --git a/docs/src/main/md/bigquery.md b/docs/src/main/md/bigquery.md index e73cbb6cc3..34401ade29 100644 --- a/docs/src/main/md/bigquery.md +++ b/docs/src/main/md/bigquery.md @@ -33,13 +33,15 @@ Gradle coordinates: The following application properties may be configured with Spring Cloud GCP BigQuery libraries. -| | | | | -| ------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Name | Description | Required | Default value | -| `spring.cloud.gcp.bigquery.datasetName` | The BigQuery dataset that the `BigQueryTemplate` and `BigQueryFileMessageHandler` is scoped to. | Yes | | -| `spring.cloud.gcp.bigquery.enabled` | Enables or disables Spring Cloud GCP BigQuery autoconfiguration. | No | `true` | -| `spring.cloud.gcp.bigquery.project-id` | GCP project ID of the project using BigQuery APIs, if different from the one in the [Spring Cloud GCP Core Module](#spring-cloud-gcp-core). | No | Project ID is typically inferred from [`gcloud`](https://cloud.google.com/sdk/gcloud/reference/config/set) configuration. | -| `spring.cloud.gcp.bigquery.credentials.location` | Credentials file location for authenticating with the Google Cloud BigQuery APIs, if different from the ones in the [Spring Cloud GCP Core Module](#spring-cloud-gcp-core) | No | Inferred from [Application Default Credentials](https://cloud.google.com/docs/authentication/production), typically set by [`gcloud`](https://cloud.google.com/sdk/gcloud/reference/auth/application-default). | +| | | | | +| ------------------------------------------------ |------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Name | Description | Required | Default value | +| `spring.cloud.gcp.bigquery.datasetName` | The BigQuery dataset that the `BigQueryTemplate` and `BigQueryFileMessageHandler` is scoped to. | Yes | | +| `spring.cloud.gcp.bigquery.enabled` | Enables or disables Spring Cloud GCP BigQuery autoconfiguration. | No | `true` | +| `spring.cloud.gcp.bigquery.project-id` | GCP project ID of the project using BigQuery APIs, if different from the one in the [Spring Cloud GCP Core Module](#spring-cloud-gcp-core). | No | Project ID is typically inferred from [`gcloud`](https://cloud.google.com/sdk/gcloud/reference/config/set) configuration. | +| `spring.cloud.gcp.bigquery.credentials.location` | Credentials file location for authenticating with the Google Cloud BigQuery APIs, if different from the ones in the [Spring Cloud GCP Core Module](#spring-cloud-gcp-core) | No | Inferred from [Application Default Credentials](https://cloud.google.com/docs/authentication/production), typically set by [`gcloud`](https://cloud.google.com/sdk/gcloud/reference/auth/application-default). | +| `spring.cloud.gcp.bigquery.jsonWriterBatchSize` | Batch size which will be used by `BigQueryJsonDataWriter` while using [BigQuery Storage Write API](https://cloud.google.com/bigquery/docs/write-api). Note too large or too low values might impact performance. | No | 1000 | +| `spring.cloud.gcp.bigquery.threadPoolSize` | The size of thread pool of `ThreadPoolTaskScheduler` which is used by `BigQueryTemplate` | No | 4 | #### BigQuery Client Object @@ -101,6 +103,58 @@ public void loadData(InputStream dataInputStream, String tableName) { } ``` +Below is a code snippet of how to load a [newline-delimited JSON](https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-json) data `InputStream` to a BigQuery table. This implementation uses the [BigQuery Storage Write API](https://cloud.google.com/bigquery/docs/write-api). +[Here](https://github.com/GoogleCloudPlatform/spring-cloud-gcp/tree/main/spring-cloud-gcp-bigquery/src/test/resources/data.json) is a sample newline-delimited JSON file which can be used for testing this functionality. + +``` java +// BigQuery client object provided by our autoconfiguration. +@Autowired +BigQueryTemplate bigQueryTemplate; + + /** + * This method loads the InputStream of the newline-delimited JSON records to be written in the given table. + * @param tableName name of the table where the data is expected to be written + * @param jsonInputStream InputStream of the newline-delimited JSON records to be written in the given table + */ + public void loadJsonStream(String tableName, InputStream jsonInputStream) + throws ExecutionException, InterruptedException { + ListenableFuture writeApFuture = + bigQueryTemplate.writeJsonStream(tableName, jsonInputStream); + WriteApiResponse apiRes = writeApFuture.get();//get the WriteApiResponse + if (!apiRes.isSuccessful()){ + List errors = apiRes.getErrors(); + //TODO(developer): process the List of StorageError + } + //else the write process has been successful + } +``` +Below is a code snippet of how to create table and then load a [newline-delimited JSON](https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-json) data `InputStream` to a BigQuery table. This implementation uses the [BigQuery Storage Write API](https://cloud.google.com/bigquery/docs/write-api). +[Here](https://github.com/GoogleCloudPlatform/spring-cloud-gcp/tree/main/spring-cloud-gcp-bigquery/src/test/resources/data.json) is a sample newline-delimited JSON file which can be used for testing this functionality. + +``` java +// BigQuery client object provided by our autoconfiguration. +@Autowired +BigQueryTemplate bigQueryTemplate; + + /** + * This method created a table with the given name and schema and then loads the InputStream of the newline-delimited JSON records in it. + * @param tableName name of the table where the data is expected to be written + * @param jsonInputStream InputStream of the newline-delimited JSON records to be written in the given table + * @param tableSchema Schema of the table which is required to be created + */ + public void createTableAndloadJsonStream(String tableName, InputStream jsonInputStream, Schema tableSchema) + throws ExecutionException, InterruptedException { + ListenableFuture writeApFuture = + bigQueryTemplate.writeJsonStream(tableName, jsonInputStream, tableSchema);//using the overloaded method which created the table when tableSchema is passed + WriteApiResponse apiRes = writeApFuture.get();//get the WriteApiResponse + if (!apiRes.isSuccessful()){ + List errors = apiRes.getErrors(); + //TODO(developer): process the List of StorageError + } + //else the write process has been successful + } +``` + ### Spring Integration Spring Cloud GCP BigQuery also provides a Spring Integration message