From 7529d7de335a461dbb65798b2a110bf8ed84447b Mon Sep 17 00:00:00 2001 From: shortcuts Date: Wed, 11 Dec 2024 11:51:08 +0100 Subject: [PATCH 1/5] guides: enable watcher for push --- tests/CTS/guides/ingestion.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/CTS/guides/ingestion.json b/tests/CTS/guides/ingestion.json index 618d1df174..dedcfb67cd 100644 --- a/tests/CTS/guides/ingestion.json +++ b/tests/CTS/guides/ingestion.json @@ -6,7 +6,8 @@ "pushTaskPayload": { "action": "addObject", "records": "$var: records" - } + }, + "watch": true } } } From 02b6103965c71fb7b432ca2f0e34fb496fdece07 Mon Sep 17 00:00:00 2001 From: shortcuts Date: Wed, 11 Dec 2024 13:46:01 +0100 Subject: [PATCH 2/5] fix: templates --- specs/ingestion/common/schemas/task.yml | 3 +++ specs/ingestion/paths/tasks/v2/pushTask.yml | 2 +- templates/csharp/guides/ingestion/pushSetup.mustache | 6 ++++-- templates/go/guides/ingestion/pushSetup.mustache | 6 +++--- templates/java/guides/ingestion/pushSetup.mustache | 6 +++--- templates/javascript/guides/ingestion/pushSetup.mustache | 7 +++---- templates/kotlin/guides/ingestion/pushSetup.mustache | 6 +++--- templates/php/guides/ingestion/pushSetup.mustache | 6 +++--- templates/python/guides/ingestion/pushSetup.mustache | 6 +++--- templates/ruby/guides/ingestion/pushSetup.mustache | 6 +++--- templates/scala/guides/ingestion/pushSetup.mustache | 6 +++--- templates/swift/guides/ingestion/pushSetup.mustache | 6 +++--- 12 files changed, 35 insertions(+), 31 deletions(-) diff --git a/specs/ingestion/common/schemas/task.yml b/specs/ingestion/common/schemas/task.yml index fdaf54114b..0bcd7d42b3 100644 --- a/specs/ingestion/common/schemas/task.yml +++ b/specs/ingestion/common/schemas/task.yml @@ -553,3 +553,6 @@ ShopifyMarket: - countries - currencies - locales + +PushTaskResponse: + $ref: './source.yml#/SourceWatchResponse' diff --git a/specs/ingestion/paths/tasks/v2/pushTask.yml b/specs/ingestion/paths/tasks/v2/pushTask.yml index 2f26178bcf..f8d328579f 100644 --- a/specs/ingestion/paths/tasks/v2/pushTask.yml +++ b/specs/ingestion/paths/tasks/v2/pushTask.yml @@ -47,6 +47,6 @@ post: content: application/json: schema: - $ref: '../../../common/schemas/run.yml#/RunResponse' + $ref: '../../common/schemas/source.yml#/SourceWatchResponse' '400': $ref: '../../../../common/responses/BadRequest.yml' diff --git a/templates/csharp/guides/ingestion/pushSetup.mustache b/templates/csharp/guides/ingestion/pushSetup.mustache index 761c1e6a36..458b19c754 100644 --- a/templates/csharp/guides/ingestion/pushSetup.mustache +++ b/templates/csharp/guides/ingestion/pushSetup.mustache @@ -20,8 +20,10 @@ class PushSetup try { - var run = {{#dynamicSnippet}}pushSetup{{/dynamicSnippet}}; - Console.WriteLine(run.RunID); + // setting `watch` to `true` will make the call synchronous + var resp = {{#dynamicSnippet}}pushSetup{{/dynamicSnippet}}; + + Console.WriteLine(resp); } catch (Exception e) { diff --git a/templates/go/guides/ingestion/pushSetup.mustache b/templates/go/guides/ingestion/pushSetup.mustache index faed3666ef..ca14cb4968 100644 --- a/templates/go/guides/ingestion/pushSetup.mustache +++ b/templates/go/guides/ingestion/pushSetup.mustache @@ -26,11 +26,11 @@ func push() { panic(err) } - run, err := {{#dynamicSnippet}}pushSetup{{/dynamicSnippet}} + // setting `watch` to `true` will make the call synchronous + resp, err := {{#dynamicSnippet}}pushSetup{{/dynamicSnippet}} if err != nil { panic(err) } - // use runID in the Observability debugger - fmt.Println("run", run.RunID) + fmt.Printf("%#v\n", resp) } \ No newline at end of file diff --git a/templates/java/guides/ingestion/pushSetup.mustache b/templates/java/guides/ingestion/pushSetup.mustache index 57e5f30289..e5f0b10aa9 100644 --- a/templates/java/guides/ingestion/pushSetup.mustache +++ b/templates/java/guides/ingestion/pushSetup.mustache @@ -13,10 +13,10 @@ public class pushSetup { // use the region matching your applicationID {{> snippets/init}} - RunResponse run = {{#dynamicSnippet}}pushSetup{{/dynamicSnippet}}; + // setting `watch` to `true` will make the call synchronous + PushTaskResponse resp = {{#dynamicSnippet}}pushSetup{{/dynamicSnippet}}; - // use runID in the Observability debugger - System.out.println(run.getRunID()); + System.out.println(resp); client.close(); } diff --git a/templates/javascript/guides/ingestion/pushSetup.mustache b/templates/javascript/guides/ingestion/pushSetup.mustache index fae35e3967..8eacd33951 100644 --- a/templates/javascript/guides/ingestion/pushSetup.mustache +++ b/templates/javascript/guides/ingestion/pushSetup.mustache @@ -10,11 +10,10 @@ try { // read local JSON file containing array of records const records = JSON.parse(fs.readFileSync('records.json', 'utf8')) as PushTaskRecords[]; - // push records to the API - const run = {{#dynamicSnippet}}pushSetup{{/dynamicSnippet}} + // setting `watch` to `true` will make the call synchronous + const resp = {{#dynamicSnippet}}pushSetup{{/dynamicSnippet}} - // use runID in the Observability debugger - console.log(run.runID); + console.log(resp); } catch (err) { console.error(err); } \ No newline at end of file diff --git a/templates/kotlin/guides/ingestion/pushSetup.mustache b/templates/kotlin/guides/ingestion/pushSetup.mustache index e3d38104be..91e42c9a5d 100644 --- a/templates/kotlin/guides/ingestion/pushSetup.mustache +++ b/templates/kotlin/guides/ingestion/pushSetup.mustache @@ -18,10 +18,10 @@ suspend fun main() { {{> snippets/init}} try { - val run = client.{{#dynamicSnippet}}pushSetup{{/dynamicSnippet}} + // setting `watch` to `true` will make the call synchronous + val resp = client.{{#dynamicSnippet}}pushSetup{{/dynamicSnippet}} - // use runID in the Observability debugger - println(run) + println(resp) } catch(e: Exception) { println(e.message) } diff --git a/templates/php/guides/ingestion/pushSetup.mustache b/templates/php/guides/ingestion/pushSetup.mustache index 60e2161117..fce0875750 100644 --- a/templates/php/guides/ingestion/pushSetup.mustache +++ b/templates/php/guides/ingestion/pushSetup.mustache @@ -7,7 +7,7 @@ $records = json_decode(file_get_contents("records.json"), true); // use the region matching your applicationID {{> snippets/init}} -$run = {{#dynamicSnippet}}pushSetup{{/dynamicSnippet}}; +// setting `watch` to `true` will make the call synchronous +$resp = {{#dynamicSnippet}}pushSetup{{/dynamicSnippet}}; -// use runID in the Observability debugger -print($run); \ No newline at end of file +print($resp); \ No newline at end of file diff --git a/templates/python/guides/ingestion/pushSetup.mustache b/templates/python/guides/ingestion/pushSetup.mustache index 3a039258a6..a647314722 100644 --- a/templates/python/guides/ingestion/pushSetup.mustache +++ b/templates/python/guides/ingestion/pushSetup.mustache @@ -10,9 +10,9 @@ async def main(): with open("records.json") as f: records = json.load(f) - run = {{#dynamicSnippet}}pushSetup{{/dynamicSnippet}} + # setting `watch` to `true` will make the call synchronous + resp = {{#dynamicSnippet}}pushSetup{{/dynamicSnippet}} - # use runID in the Observability debugger - print(run.run_id) + print(resp) asyncio.run(main()) \ No newline at end of file diff --git a/templates/ruby/guides/ingestion/pushSetup.mustache b/templates/ruby/guides/ingestion/pushSetup.mustache index 85742c6275..9282f13d60 100644 --- a/templates/ruby/guides/ingestion/pushSetup.mustache +++ b/templates/ruby/guides/ingestion/pushSetup.mustache @@ -6,7 +6,7 @@ records = JSON.parse(File.read('records.json')) # use the region matching your applicationID {{> snippets/init}} -run = {{#dynamicSnippet}}pushSetup{{/dynamicSnippet}} +# setting `watch` to `true` will make the call synchronous +resp = {{#dynamicSnippet}}pushSetup{{/dynamicSnippet}} -# use runID in the Observability debugger -puts run.run_id \ No newline at end of file +puts resp \ No newline at end of file diff --git a/templates/scala/guides/ingestion/pushSetup.mustache b/templates/scala/guides/ingestion/pushSetup.mustache index 6d4784f3b8..cf810076e1 100644 --- a/templates/scala/guides/ingestion/pushSetup.mustache +++ b/templates/scala/guides/ingestion/pushSetup.mustache @@ -21,13 +21,13 @@ object PushSetup { {{> snippets/init}} try { - val run = Await.result( + // setting `watch` to `true` will make the call synchronous + val resp = Await.result( {{#dynamicSnippet}}pushSetup{{/dynamicSnippet}}, Duration(100, "sec") ) - // use runID in the Observability debugger - println(run.runID) + println(resp) } catch { case e: Exception => println(e) } diff --git a/templates/swift/guides/ingestion/pushSetup.mustache b/templates/swift/guides/ingestion/pushSetup.mustache index 4b5f529691..ce2a5e6d4f 100644 --- a/templates/swift/guides/ingestion/pushSetup.mustache +++ b/templates/swift/guides/ingestion/pushSetup.mustache @@ -16,10 +16,10 @@ func pushSetup() async throws { // use the region matching your applicationID {{> snippets/init}} - let run = {{#dynamicSnippet}}pushSetup{{/dynamicSnippet}} + // setting `watch` to `true` will make the call synchronous + let resp = {{#dynamicSnippet}}pushSetup{{/dynamicSnippet}} - // use runID in the Observability debugger - dump(run) + dump(resp) } catch { print(error) } From 4cd0bc181819aac4c4e6a7a6198efed7b5ef0a93 Mon Sep 17 00:00:00 2001 From: shortcuts Date: Wed, 11 Dec 2024 13:57:06 +0100 Subject: [PATCH 3/5] fix: response --- specs/ingestion/paths/tasks/v2/pushTask.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/ingestion/paths/tasks/v2/pushTask.yml b/specs/ingestion/paths/tasks/v2/pushTask.yml index f8d328579f..112e855835 100644 --- a/specs/ingestion/paths/tasks/v2/pushTask.yml +++ b/specs/ingestion/paths/tasks/v2/pushTask.yml @@ -47,6 +47,6 @@ post: content: application/json: schema: - $ref: '../../common/schemas/source.yml#/SourceWatchResponse' + $ref: '../../../common/schemas/task.yml#/PushTaskResponse' '400': $ref: '../../../../common/responses/BadRequest.yml' From 8103fd76cf4d963a9083364bf74b4e46c43d01de Mon Sep 17 00:00:00 2001 From: shortcuts Date: Wed, 11 Dec 2024 14:08:18 +0100 Subject: [PATCH 4/5] trigger From 80cc695ccddd25d68121bb8a9b4c5c49cbd9e6fd Mon Sep 17 00:00:00 2001 From: shortcuts Date: Wed, 11 Dec 2024 14:53:22 +0100 Subject: [PATCH 5/5] fix: WatchResponse --- specs/ingestion/common/schemas/common.yml | 22 +++++++++++++++++++ specs/ingestion/common/schemas/source.yml | 22 ------------------- specs/ingestion/common/schemas/task.yml | 3 --- specs/ingestion/paths/sources/discover.yml | 2 +- specs/ingestion/paths/sources/validate.yml | 2 +- specs/ingestion/paths/sources/validateID.yml | 2 +- specs/ingestion/paths/tasks/v2/pushTask.yml | 2 +- .../java/guides/ingestion/pushSetup.mustache | 2 +- 8 files changed, 27 insertions(+), 30 deletions(-) diff --git a/specs/ingestion/common/schemas/common.yml b/specs/ingestion/common/schemas/common.yml index ddf2dd9d3a..f06c607fa8 100644 --- a/specs/ingestion/common/schemas/common.yml +++ b/specs/ingestion/common/schemas/common.yml @@ -96,3 +96,25 @@ Window: required: - startDate - endDate + +WatchResponse: + type: object + additionalProperties: false + properties: + runID: + $ref: './common.yml#/runID' + data: + type: array + description: when used with discovering or validating sources, the sampled data of your source is returned. + items: + type: object + events: + description: in case of error, observability events will be added to the response, if any. + type: array + items: + $ref: '../../common/schemas/event.yml#/Event' + message: + description: a message describing the outcome of a validate run. + type: string + required: + - message diff --git a/specs/ingestion/common/schemas/source.yml b/specs/ingestion/common/schemas/source.yml index f15974efed..b85f6abdd5 100644 --- a/specs/ingestion/common/schemas/source.yml +++ b/specs/ingestion/common/schemas/source.yml @@ -476,28 +476,6 @@ sourceShopifyBase: x-discriminator-fields: - shopURL -SourceWatchResponse: - type: object - additionalProperties: false - properties: - runID: - $ref: './common.yml#/runID' - data: - type: array - description: depending on the source type, the validation returns sampling data of your source (JSON, CSV, BigQuery). - items: - type: object - events: - description: in case of error, observability events will be added to the response, if any. - type: array - items: - $ref: '../../common/schemas/event.yml#/Event' - message: - description: a message describing the outcome of a validate run. - type: string - required: - - message - RunSourcePayload: type: object additionalProperties: false diff --git a/specs/ingestion/common/schemas/task.yml b/specs/ingestion/common/schemas/task.yml index 0bcd7d42b3..fdaf54114b 100644 --- a/specs/ingestion/common/schemas/task.yml +++ b/specs/ingestion/common/schemas/task.yml @@ -553,6 +553,3 @@ ShopifyMarket: - countries - currencies - locales - -PushTaskResponse: - $ref: './source.yml#/SourceWatchResponse' diff --git a/specs/ingestion/paths/sources/discover.yml b/specs/ingestion/paths/sources/discover.yml index 4a66698601..4e1cc387ed 100644 --- a/specs/ingestion/paths/sources/discover.yml +++ b/specs/ingestion/paths/sources/discover.yml @@ -18,6 +18,6 @@ post: content: application/json: schema: - $ref: '../../common/schemas/source.yml#/SourceWatchResponse' + $ref: '../../common/schemas/common.yml#/WatchResponse' '400': $ref: '../../../common/responses/BadRequest.yml' diff --git a/specs/ingestion/paths/sources/validate.yml b/specs/ingestion/paths/sources/validate.yml index dd3ede67ea..447f65b1a0 100644 --- a/specs/ingestion/paths/sources/validate.yml +++ b/specs/ingestion/paths/sources/validate.yml @@ -21,6 +21,6 @@ post: content: application/json: schema: - $ref: '../../common/schemas/source.yml#/SourceWatchResponse' + $ref: '../../common/schemas/common.yml#/WatchResponse' '400': $ref: '../../../common/responses/BadRequest.yml' diff --git a/specs/ingestion/paths/sources/validateID.yml b/specs/ingestion/paths/sources/validateID.yml index c00f89502f..f40028bc4f 100644 --- a/specs/ingestion/paths/sources/validateID.yml +++ b/specs/ingestion/paths/sources/validateID.yml @@ -23,6 +23,6 @@ post: content: application/json: schema: - $ref: '../../common/schemas/source.yml#/SourceWatchResponse' + $ref: '../../common/schemas/common.yml#/WatchResponse' '400': $ref: '../../../common/responses/BadRequest.yml' diff --git a/specs/ingestion/paths/tasks/v2/pushTask.yml b/specs/ingestion/paths/tasks/v2/pushTask.yml index 112e855835..b43c64d5d6 100644 --- a/specs/ingestion/paths/tasks/v2/pushTask.yml +++ b/specs/ingestion/paths/tasks/v2/pushTask.yml @@ -47,6 +47,6 @@ post: content: application/json: schema: - $ref: '../../../common/schemas/task.yml#/PushTaskResponse' + $ref: '../../../common/schemas/common.yml#/WatchResponse' '400': $ref: '../../../../common/responses/BadRequest.yml' diff --git a/templates/java/guides/ingestion/pushSetup.mustache b/templates/java/guides/ingestion/pushSetup.mustache index e5f0b10aa9..e133154c56 100644 --- a/templates/java/guides/ingestion/pushSetup.mustache +++ b/templates/java/guides/ingestion/pushSetup.mustache @@ -14,7 +14,7 @@ public class pushSetup { {{> snippets/init}} // setting `watch` to `true` will make the call synchronous - PushTaskResponse resp = {{#dynamicSnippet}}pushSetup{{/dynamicSnippet}}; + WatchResponse resp = {{#dynamicSnippet}}pushSetup{{/dynamicSnippet}}; System.out.println(resp);