From e1144024c0b0f61fde02e658e1404540f0219dac Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 14 Feb 2024 17:12:39 +0200 Subject: [PATCH 01/33] added source.py --- .../source-recharge/source_recharge/source.py | 70 +++---------------- 1 file changed, 10 insertions(+), 60 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/source.py b/airbyte-integrations/connectors/source-recharge/source_recharge/source.py index 1d1ea875f3e3..2dd2f2794439 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/source.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/source.py @@ -2,66 +2,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from typing import Any, List, Mapping, Tuple, Union +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -from .api import ( - Addresses, - Charges, - Collections, - Customers, - Discounts, - Metafields, - Onetimes, - OrdersDeprecatedApi, - OrdersModernApi, - Products, - Shop, - Subscriptions, -) - - -class RechargeTokenAuthenticator(TokenAuthenticator): - def get_auth_header(self) -> Mapping[str, Any]: - return {"X-Recharge-Access-Token": self._token} - - -class SourceRecharge(AbstractSource): - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: - auth = RechargeTokenAuthenticator(token=config["access_token"]) - stream = Shop(config, authenticator=auth) - try: - result = list(stream.read_records(SyncMode.full_refresh))[0] - if stream.name in result.keys(): - return True, None - except Exception as error: - return False, f"Unable to connect to Recharge API with the provided credentials - {repr(error)}" - - def select_orders_stream(self, config: Mapping[str, Any], **kwargs) -> Union[OrdersDeprecatedApi, OrdersModernApi]: - if config.get("use_orders_deprecated_api"): - return OrdersDeprecatedApi(config, **kwargs) - else: - return OrdersModernApi(config, **kwargs) - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - auth = RechargeTokenAuthenticator(token=config["access_token"]) - return [ - Addresses(config, authenticator=auth), - Charges(config, authenticator=auth), - Collections(config, authenticator=auth), - Customers(config, authenticator=auth), - Discounts(config, authenticator=auth), - Metafields(config, authenticator=auth), - Onetimes(config, authenticator=auth), - # select the Orders stream class, based on the UI toggle "Use `Orders` Deprecated API" - self.select_orders_stream(config, authenticator=auth), - Products(config, authenticator=auth), - Shop(config, authenticator=auth), - Subscriptions(config, authenticator=auth), - ] +# Declarative Source +class SourceRecharge(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) From dc9b4eefe4e2f11a82ff2981b6b8cea5af15ab93 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 15 Feb 2024 16:36:28 +0200 Subject: [PATCH 02/33] added Products stream --- .../source_recharge/manifest.yaml | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml b/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml new file mode 100644 index 000000000000..f826d3bb4939 --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml @@ -0,0 +1,126 @@ +version: 0.58.4 + +# COMMON DEFINITIONS +definitions: + + # API VERSION REFERENCE + modern_api_version: + x-recharge-version: "2021-11" + deprecated_api_version: + x-recharge-version: "2021-01" + + # COMMON PARTS + schema_loader: + type: JsonFileSchemaLoader + file_path: "./source_recharge/schemas/{{ parameters['name'] }}.json" + selector: + description: >- + Base records selector for Full Refresh streams + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["{{ parameters.get('data_path')}}"] + # apply default schema normalization + schema_normalization: Default + authenticator: + type: ApiKeyAuthenticator + api_token: "{{ config['access_token'] }}" + inject_into: + type: RequestOption + inject_into: header + field_name: X-Recharge-Access-Token + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + inject_into: request_parameter + field_name: limit + type: RequestOption + pagination_strategy: + type: PageIncrement + start_from_page: 1 + page_size: 250 + inject_on_first_request: false + + # REQUESTERS + default_requester: + description: >- + Default Base Requester for Full Refresh streams + type: HttpRequester + url_base: https://api.rechargeapps.com/ + path: "{{ parameters['name'] }}" + http_method: GET + authenticator: + $ref: "#/definitions/authenticator" + error_handler: + type: "DefaultErrorHandler" + # DEFAULT REQUESTER FOR MODERN API + default_retriever: + description: >- + Base Deprecated Recharge API Retriever for Full Refresh streams. + Doc: https://developer.rechargepayments.com/2021-01/versioning + record_selector: + $ref: "#/definitions/selector" + requester: + $ref: "#/definitions/default_requester" + request_headers: + $ref: "#/definitions/modern_api_version" + paginator: + $ref: "#/definitions/paginator" + + # RETRIEVER FOR `DEPRECATED API` + retriever_api_deprecated: + $ref: "#/definitions/default_retriever" + # Override to have deprecated api version calls + requester: + $ref: "#/definitions/default_requester" + request_headers: + # for deprecated retriever we should use `2021-01` api version + $ref: "#/definitions/deprecated_api_version" + # RETRIEVER FOR `SHOP` STREAM + retriever_shop_stream: + $ref: "#/definitions/retriever_api_deprecated" + # Override the selector to provide the default value + record_selector: + $ref: "#/definitions/selector" + extractor: + type: DpathExtractor + field_path: [] + + # BASE FULL-REFRESH STREAMS + base_deprecated_api_stream: + primary_key: "id" + schema_loader: + $ref: "#/definitions/schema_loader" + retriever: + $ref: "#/definitions/retriever_api_deprecated" + $parameters: + start_date: "{{ config['start_date'] }}" + raise_on_http_errors: true + + # FULL-REFRESH STREAMS + shop_stream: + $ref: "#/definitions/base_deprecated_api_stream" + retriever: + $ref: "#/definitions/retriever_shop_stream" + primary_key: ["shop", "store"] + $parameters: + name: "shop" + + products_stream: + $ref: "#/definitions/base_deprecated_api_stream" + $parameters: + name: "products" + data_path: "products" + +streams: + - "#/definitions/shop_stream" + - "#/definitions/products_stream" + +check: + type: CheckStream + stream_names: + - shop From 0b8030ce716fc30320a94abe9b815654639b3e29 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Tue, 20 Feb 2024 20:41:42 +0200 Subject: [PATCH 03/33] updated draft --- .../components/datetime_based_cursor.py | 39 +++++ .../source_recharge/manifest.yaml | 147 ++++++++++++++---- 2 files changed, 159 insertions(+), 27 deletions(-) create mode 100644 airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py b/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py new file mode 100644 index 000000000000..0bf213b881bf --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py @@ -0,0 +1,39 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from dataclasses import dataclass +from typing import Any, Mapping, Optional + +from airbyte_cdk.sources.declarative.types import Record, StreamSlice +from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor + + +@dataclass +class RechargeDateTimeBasedCursor(DatetimeBasedCursor): + """ + Override for the default `DatetimeBasedCursor` to make self.close_slice() to produce `min` value instead of `max` value. + + This is the ONLY CHANGE MADE HERE, to make the SOURCE STATE proccessed correctly: + The `min` value should be determined, in the first place, since we would skip the records, + if they are updated manually, by the Customer, and the range in not AFTER the STATE value, but before. + """ + + def __post_init__(self, parameters: Mapping[str, Any]) -> None: + super().__post_init__(parameters=parameters) + + def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None: + last_record_cursor_value = most_recent_record.get(self.cursor_field.eval(self.config)) if most_recent_record else None + stream_slice_value_end = stream_slice.get(self.partition_field_end.eval(self.config)) + cursor_value_str_by_cursor_value_datetime = dict( + map( + lambda datetime_str: (self.parse_date(datetime_str), datetime_str), + filter(lambda item: item, [self._cursor, last_record_cursor_value, stream_slice_value_end]), + ) + ) + self._cursor = ( + cursor_value_str_by_cursor_value_datetime[min(cursor_value_str_by_cursor_value_datetime.keys())] + if cursor_value_str_by_cursor_value_datetime + else None + ) diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml b/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml index f826d3bb4939..f4bf72154dde 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml @@ -1,14 +1,12 @@ version: 0.58.4 -# COMMON DEFINITIONS definitions: - # API VERSION REFERENCE - modern_api_version: - x-recharge-version: "2021-11" deprecated_api_version: x-recharge-version: "2021-01" - + modern_api_version: + x-recharge-version: "2021-11" + # COMMON PARTS schema_loader: type: JsonFileSchemaLoader @@ -22,6 +20,7 @@ definitions: field_path: ["{{ parameters.get('data_path')}}"] # apply default schema normalization schema_normalization: Default + authenticator: type: ApiKeyAuthenticator api_token: "{{ config['access_token'] }}" @@ -29,7 +28,9 @@ definitions: type: RequestOption inject_into: header field_name: X-Recharge-Access-Token - paginator: + + # PAGINATORS + paginator_deprecated_api: type: DefaultPaginator page_token_option: type: RequestOption @@ -44,42 +45,80 @@ definitions: start_from_page: 1 page_size: 250 inject_on_first_request: false + paginator_modern_api: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: cursor + page_size_option: + inject_into: request_parameter + type: RequestOption + field_name: limit + pagination_strategy: + type: CursorPagination + page_size: 1 + cursor_value: '{{ response.get("next_cursor", {}) }}' + stop_condition: '{{ not response.get("next_cursor", {}) }}' # REQUESTERS - default_requester: + requester_deprecated_api: + description: >- + Default Base Requester for Full Refresh streams + type: HttpRequester + url_base: https://api.rechargeapps.com/ + path: "{{ parameters['name'] }}" + http_method: GET + authenticator: + $ref: "#/definitions/authenticator" + request_headers: + # for deprecated retriever we should use `2021-01` api version + $ref: "#/definitions/deprecated_api_version" + error_handler: + type: "DefaultErrorHandler" + + requester_modern_api: description: >- Default Base Requester for Full Refresh streams + # TODO: + # WHEN the default HttpRequester is used, there is no option to omit passing additional + # query params along with `next_page_token`, the fix should be probably be made on the CDK lvl. type: HttpRequester url_base: https://api.rechargeapps.com/ path: "{{ parameters['name'] }}" http_method: GET authenticator: $ref: "#/definitions/authenticator" + request_headers: + # for modern retriever we should use >= `2021-11` api version + $ref: "#/definitions/modern_api_version" error_handler: type: "DefaultErrorHandler" - # DEFAULT REQUESTER FOR MODERN API - default_retriever: + request_parameters: + updated_at_min: '{{ stream_state.get(''updated_at'') if stream_state else config[''start_date''] }}' + + # RETRIEVER FOR `DEPRECATED API` + retriever_api_deprecated: description: >- - Base Deprecated Recharge API Retriever for Full Refresh streams. - Doc: https://developer.rechargepayments.com/2021-01/versioning + Default Retriever for Deprecated API `2021-01` Full Refresh streams. record_selector: $ref: "#/definitions/selector" requester: - $ref: "#/definitions/default_requester" - request_headers: - $ref: "#/definitions/modern_api_version" + $ref: "#/definitions/requester_deprecated_api" paginator: - $ref: "#/definitions/paginator" + $ref: "#/definitions/paginator_deprecated_api" - # RETRIEVER FOR `DEPRECATED API` - retriever_api_deprecated: - $ref: "#/definitions/default_retriever" - # Override to have deprecated api version calls + # RETRIEVER FOR `MODERN API` + retriever_api_modern: + description: >- + Default Retriever for Modern API `2021-11` Full Refresh streams. + record_selector: + $ref: "#/definitions/selector" requester: - $ref: "#/definitions/default_requester" - request_headers: - # for deprecated retriever we should use `2021-01` api version - $ref: "#/definitions/deprecated_api_version" + $ref: "#/definitions/requester_modern_api" + paginator: + $ref: "#/definitions/paginator_modern_api" + # RETRIEVER FOR `SHOP` STREAM retriever_shop_stream: $ref: "#/definitions/retriever_api_deprecated" @@ -97,11 +136,56 @@ definitions: $ref: "#/definitions/schema_loader" retriever: $ref: "#/definitions/retriever_api_deprecated" - $parameters: - start_date: "{{ config['start_date'] }}" - raise_on_http_errors: true + # $parameters: + # start_date: "{{ config['start_date'] }}" + # raise_on_http_errors: true + + base_modern_api_stream: + primary_key: "id" + schema_loader: + $ref: "#/definitions/schema_loader" + retriever: + $ref: "#/definitions/retriever_api_modern" + + # BASE INCREMENTAL STREAMS + base_deprecated_api_incremental_stream: + $ref: "#/definitions/base_deprecated_api_stream" + # incremental_sync: + # type: CustomIncrementalSync + # class_name: source_recharge.components.RechargeDateTimeBasedCursor + # cursor_field: "updated_at" + # cursor_datetime_formats: + # - '%Y-%m-%dT%H:%M:%S%z' + # datetime_format: '%Y-%m-%dT%H:%M:%S%z' + # start_datetime: + # type: MinMaxDatetime + # datetime: '{{ config[''start_date''] }}' + # datetime_format: '%Y-%m-%dT%H:%M:%SZ' + # start_time_option: + # inject_into: request_parameter + # type: RequestOption + # field_name: 'updated_at_min' + + base_modern_api_incremental_stream: + $ref: "#/definitions/base_modern_api_stream" + incremental_sync: + # type: DatetimeBasedCursor + # The custom incremental sync was applied because, + # the cursor for the close_slice() method is determined as NOW(), + # instead of the real cursor field (updated_at) value from the record. + type: CustomIncrementalSync + class_name: source_recharge.components.datetime_based_cursor.RechargeDateTimeBasedCursor + cursor_field: "updated_at" + cursor_datetime_formats: + - '%Y-%m-%dT%H:%M:%S%z' + datetime_format: '%Y-%m-%dT%H:%M:%S%z' + start_datetime: + type: MinMaxDatetime + datetime: '{{ config[''start_date''] }}' + datetime_format: '%Y-%m-%dT%H:%M:%SZ' # FULL-REFRESH STREAMS + # SHOP shop_stream: $ref: "#/definitions/base_deprecated_api_stream" retriever: @@ -109,16 +193,25 @@ definitions: primary_key: ["shop", "store"] $parameters: name: "shop" - + # PRODUCTS products_stream: $ref: "#/definitions/base_deprecated_api_stream" $parameters: name: "products" data_path: "products" + + # INCREMENTAL STREAMS + # ORDERS + orders_stream: + $ref: "#/definitions/base_modern_api_incremental_stream" + $parameters: + name: "orders" + data_path: "orders" streams: - "#/definitions/shop_stream" - "#/definitions/products_stream" + - "#/definitions/orders_stream" check: type: CheckStream From 88798d6615cb9de2bf28ea185bdba95ac303633c Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Tue, 20 Feb 2024 20:45:15 +0200 Subject: [PATCH 04/33] updated draft --- .../source_recharge/manifest.yaml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml b/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml index f4bf72154dde..18a11ae2b510 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml @@ -136,9 +136,6 @@ definitions: $ref: "#/definitions/schema_loader" retriever: $ref: "#/definitions/retriever_api_deprecated" - # $parameters: - # start_date: "{{ config['start_date'] }}" - # raise_on_http_errors: true base_modern_api_stream: primary_key: "id" @@ -150,26 +147,10 @@ definitions: # BASE INCREMENTAL STREAMS base_deprecated_api_incremental_stream: $ref: "#/definitions/base_deprecated_api_stream" - # incremental_sync: - # type: CustomIncrementalSync - # class_name: source_recharge.components.RechargeDateTimeBasedCursor - # cursor_field: "updated_at" - # cursor_datetime_formats: - # - '%Y-%m-%dT%H:%M:%S%z' - # datetime_format: '%Y-%m-%dT%H:%M:%S%z' - # start_datetime: - # type: MinMaxDatetime - # datetime: '{{ config[''start_date''] }}' - # datetime_format: '%Y-%m-%dT%H:%M:%SZ' - # start_time_option: - # inject_into: request_parameter - # type: RequestOption - # field_name: 'updated_at_min' base_modern_api_incremental_stream: $ref: "#/definitions/base_modern_api_stream" incremental_sync: - # type: DatetimeBasedCursor # The custom incremental sync was applied because, # the cursor for the close_slice() method is determined as NOW(), # instead of the real cursor field (updated_at) value from the record. From 6cfb2e855236516a4b0e5e7459ce71e0bef49db4 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Fri, 23 Feb 2024 15:45:27 +0200 Subject: [PATCH 05/33] added metafields and other streams, added RETRY for 200 and broken response VS Content-Length --- .../source_recharge/manifest.yaml | 220 ++++++++++++------ .../source-recharge/source_recharge/source.py | 12 +- 2 files changed, 163 insertions(+), 69 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml b/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml index 18a11ae2b510..8228265d8434 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml @@ -1,4 +1,4 @@ -version: 0.58.4 +version: 0.65.0 definitions: # API VERSION REFERENCE @@ -6,7 +6,7 @@ definitions: x-recharge-version: "2021-01" modern_api_version: x-recharge-version: "2021-11" - + # COMMON PARTS schema_loader: type: JsonFileSchemaLoader @@ -20,7 +20,6 @@ definitions: field_path: ["{{ parameters.get('data_path')}}"] # apply default schema normalization schema_normalization: Default - authenticator: type: ApiKeyAuthenticator api_token: "{{ config['access_token'] }}" @@ -57,12 +56,12 @@ definitions: field_name: limit pagination_strategy: type: CursorPagination - page_size: 1 + page_size: 250 cursor_value: '{{ response.get("next_cursor", {}) }}' stop_condition: '{{ not response.get("next_cursor", {}) }}' - + # REQUESTERS - requester_deprecated_api: + requester_base: description: >- Default Base Requester for Full Refresh streams type: HttpRequester @@ -71,32 +70,26 @@ definitions: http_method: GET authenticator: $ref: "#/definitions/authenticator" - request_headers: - # for deprecated retriever we should use `2021-01` api version - $ref: "#/definitions/deprecated_api_version" error_handler: - type: "DefaultErrorHandler" + type: DefaultErrorHandler + description: >- + We should retry the SUCCESSFULL request (200), with incomplete response.content + Case when: `data declared to receive` less than `actually received` + response_filters: + - action: RETRY + predicate: "{{ response.content|length > 0 and headers.get('Content-Length', 0)|int > response.content|length }}" + requester_deprecated_api: + $ref: "#/definitions/requester_base" + # for deprecated retriever we should use `2021-01` api version + request_headers: + $ref: "#/definitions/deprecated_api_version" requester_modern_api: - description: >- - Default Base Requester for Full Refresh streams - # TODO: - # WHEN the default HttpRequester is used, there is no option to omit passing additional - # query params along with `next_page_token`, the fix should be probably be made on the CDK lvl. - type: HttpRequester - url_base: https://api.rechargeapps.com/ - path: "{{ parameters['name'] }}" - http_method: GET - authenticator: - $ref: "#/definitions/authenticator" + $ref: "#/definitions/requester_base" + # for modern retriever we should use >= `2021-11` api version request_headers: - # for modern retriever we should use >= `2021-11` api version $ref: "#/definitions/modern_api_version" - error_handler: - type: "DefaultErrorHandler" - request_parameters: - updated_at_min: '{{ stream_state.get(''updated_at'') if stream_state else config[''start_date''] }}' - + # RETRIEVER FOR `DEPRECATED API` retriever_api_deprecated: description: >- @@ -107,7 +100,6 @@ definitions: $ref: "#/definitions/requester_deprecated_api" paginator: $ref: "#/definitions/paginator_deprecated_api" - # RETRIEVER FOR `MODERN API` retriever_api_modern: description: >- @@ -118,42 +110,49 @@ definitions: $ref: "#/definitions/requester_modern_api" paginator: $ref: "#/definitions/paginator_modern_api" - - # RETRIEVER FOR `SHOP` STREAM - retriever_shop_stream: - $ref: "#/definitions/retriever_api_deprecated" - # Override the selector to provide the default value - record_selector: - $ref: "#/definitions/selector" - extractor: - type: DpathExtractor - field_path: [] - - # BASE FULL-REFRESH STREAMS - base_deprecated_api_stream: + # we should ignore all other req.params once we have the `next_page_token` in response + # for pagination in `2021-11` - modern api. + ignore_stream_slicer_parameters_on_paginated_requests: true + # RETRIEVER FOR `METAFIELDS` STREAM + retriever_metafields: + $ref: "#/definitions/retriever_api_modern" + partition_router: + type: ListPartitionRouter + cursor_field: owner_resource + values: + - address + - order + - charge + - customer + - store + - subscription + request_option: + inject_into: request_parameter + type: RequestOption + field_name: owner_resource + # requester: + # $ref: "#/definitions/requester_modern_api" + # request_parameters: + # owner_resource: "{{ stream_partition['owner_resource'] }}" + + # BASE STREAMS + # FULL-REFRESH + base_stream: primary_key: "id" schema_loader: $ref: "#/definitions/schema_loader" + base_deprecated_api_stream: + $ref: "#/definitions/base_stream" retriever: $ref: "#/definitions/retriever_api_deprecated" - base_modern_api_stream: - primary_key: "id" - schema_loader: - $ref: "#/definitions/schema_loader" + $ref: "#/definitions/base_stream" retriever: $ref: "#/definitions/retriever_api_modern" - - # BASE INCREMENTAL STREAMS - base_deprecated_api_incremental_stream: - $ref: "#/definitions/base_deprecated_api_stream" - - base_modern_api_incremental_stream: + # INCREMENTAL + base_incremental_stream: $ref: "#/definitions/base_modern_api_stream" incremental_sync: - # The custom incremental sync was applied because, - # the cursor for the close_slice() method is determined as NOW(), - # instead of the real cursor field (updated_at) value from the record. type: CustomIncrementalSync class_name: source_recharge.components.datetime_based_cursor.RechargeDateTimeBasedCursor cursor_field: "updated_at" @@ -164,35 +163,120 @@ definitions: type: MinMaxDatetime datetime: '{{ config[''start_date''] }}' datetime_format: '%Y-%m-%dT%H:%M:%SZ' + start_time_option: + type: RequestOption + field_name: "updated_at_min" + inject_into: request_parameter # FULL-REFRESH STREAMS + # COLLECTIONS + collections_stream: + description: >- + Collections Stream: https://developer.rechargepayments.com/2021-11/collections/collections_list + $ref: "#/definitions/base_modern_api_stream" + $parameters: + name: "collections" + data_path: "collections" + # METAFIELDS + metafields_stream: + description: >- + Metafields Stream: https://developer.rechargepayments.com/2021-11/metafields + $ref: "#/definitions/base_modern_api_stream" + retriever: + $ref: "#/definitions/retriever_metafields" + $parameters: + name: "metafields" + data_path: "metafields" + # PRODUCTS + products_stream: + description: >- + Products Stream: https://developer.rechargepayments.com/2021-11/products/products_list + Products endpoint has 422 error with 2021-11 API version + $ref: "#/definitions/base_deprecated_api_stream" + $parameters: + name: "products" + data_path: "products" # SHOP shop_stream: + description: >- + Shop Stream: https://developer.rechargepayments.com/v1-shopify?python#shop + Shop endpoint is not available in 2021-11 API version $ref: "#/definitions/base_deprecated_api_stream" retriever: - $ref: "#/definitions/retriever_shop_stream" + $ref: "#/definitions/retriever_api_deprecated" + record_selector: + $ref: "#/definitions/selector" + extractor: + type: DpathExtractor + field_path: [] primary_key: ["shop", "store"] $parameters: name: "shop" - # PRODUCTS - products_stream: - $ref: "#/definitions/base_deprecated_api_stream" - $parameters: - name: "products" - data_path: "products" # INCREMENTAL STREAMS - # ORDERS - orders_stream: - $ref: "#/definitions/base_modern_api_incremental_stream" + # ADDRESSES + addresses_stream: + description: >- + Addresses Stream: https://developer.rechargepayments.com/2021-11/addresses/list_addresses + $ref: "#/definitions/base_incremental_stream" + $parameters: + name: "addresses" + data_path: "addresses" + # CHARGES + charges_stream: + description: >- + Charges Stream: https://developer.rechargepayments.com/2021-11/charges/charge_list + $ref: "#/definitions/base_incremental_stream" $parameters: - name: "orders" - data_path: "orders" + name: "charges" + data_path: "charges" + # CUSTOMERS + customers_stream: + description: >- + Customers Stream: https://developer.rechargepayments.com/2021-11/customers/customers_list + $ref: "#/definitions/base_incremental_stream" + $parameters: + name: "customers" + data_path: "customers" + # DISCOUNTS + discounts_stream: + description: >- + Discounts Stream: https://developer.rechargepayments.com/2021-11/discounts/discounts_list + $ref: "#/definitions/base_incremental_stream" + $parameters: + name: "discounts" + data_path: "discounts" + # ONETIMES + onetimes_stream: + description: >- + Onetimes Stream: https://developer.rechargepayments.com/2021-11/onetimes/onetimes_list + $ref: "#/definitions/base_incremental_stream" + $parameters: + name: "onetimes" + data_path: "onetimes" + # SUBSCRIPTIONS + subscriptions_stream: + # description: >- + # Subscriptions Stream: https://developer.rechargepayments.com/2021-11/subscriptions/subscriptions_list + $ref: "#/definitions/base_incremental_stream" + $parameters: + name: "subscriptions" + data_path: "subscriptions" streams: - - "#/definitions/shop_stream" + - "#/definitions/addresses_stream" + - "#/definitions/charges_stream" + - "#/definitions/collections_stream" + - "#/definitions/customers_stream" + - "#/definitions/discounts_stream" + - "#/definitions/metafields_stream" + - "#/definitions/onetimes_stream" - "#/definitions/products_stream" - - "#/definitions/orders_stream" + - "#/definitions/shop_stream" + - "#/definitions/subscriptions_stream" + # The `orders` stream remains implemented in `streams.py` due to: + # 1. Inability to resolve `$ref` conditionally + # 2. Inability to dynamically switch between paginators (diff api versions, require diff pagination approach) (or create the CustomPaginator component) check: type: CheckStream diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/source.py b/airbyte-integrations/connectors/source-recharge/source_recharge/source.py index 2dd2f2794439..f2f5d0c873c3 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/source.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/source.py @@ -2,7 +2,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + +from typing import Any, List, Mapping +from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource +from source_recharge.streams import Orders, RechargeTokenAuthenticator """ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into @@ -13,5 +17,11 @@ # Declarative Source class SourceRecharge(YamlDeclarativeSource): - def __init__(self): + def __init__(self) -> None: super().__init__(**{"path_to_yaml": "manifest.yaml"}) + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + auth = RechargeTokenAuthenticator(token=config["access_token"]) + streams = super().streams(config=config) + streams.append(Orders(config, authenticator=auth)) + return streams From 95bf6a0cec0b8718d91f3cd4657f7839d76e9d88 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Fri, 23 Feb 2024 18:17:50 +0200 Subject: [PATCH 06/33] updated poetry lock, formated --- .../integration_tests/configured_catalog.json | 13 ------- .../connectors/source-recharge/poetry.lock | 38 +++++++++---------- .../connectors/source-recharge/pyproject.toml | 2 +- .../components/datetime_based_cursor.py | 4 +- .../source_recharge/manifest.yaml | 33 +++++++--------- .../source-recharge/source_recharge/source.py | 3 +- 6 files changed, 37 insertions(+), 56 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-recharge/integration_tests/configured_catalog.json index 279ceb19c7e3..b08c61a13137 100644 --- a/airbyte-integrations/connectors/source-recharge/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-recharge/integration_tests/configured_catalog.json @@ -108,19 +108,6 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, - { - "stream": { - "name": "subscriptions", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["updated_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["updated_at"] - }, { "stream": { "name": "metafields", diff --git a/airbyte-integrations/connectors/source-recharge/poetry.lock b/airbyte-integrations/connectors/source-recharge/poetry.lock index 4135398b392f..bfd053afdb5b 100644 --- a/airbyte-integrations/connectors/source-recharge/poetry.lock +++ b/airbyte-integrations/connectors/source-recharge/poetry.lock @@ -2,13 +2,13 @@ [[package]] name = "airbyte-cdk" -version = "0.60.1" +version = "0.65.0" description = "A framework for writing Airbyte Connectors." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte-cdk-0.60.1.tar.gz", hash = "sha256:fc5212b2962c1dc6aca9cc6f1c2000d7636b7509915846c126420c2b0c814317"}, - {file = "airbyte_cdk-0.60.1-py3-none-any.whl", hash = "sha256:94b33c0f6851d1e2546eac3cec54c67489239595d9e0a496ef57c3fc808e89e3"}, + {file = "airbyte-cdk-0.65.0.tar.gz", hash = "sha256:068d4419227f27d9e1d9429f27c4160850a6d6208417828055b89d1f4ef3c367"}, + {file = "airbyte_cdk-0.65.0-py3-none-any.whl", hash = "sha256:8436125d39bd058ee0ffa67d47304ab5c707b3e184dd8fcd07848959aa3b0efc"}, ] [package.dependencies] @@ -808,13 +808,13 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requests-cache" -version = "1.1.1" +version = "1.2.0" description = "A persistent cache for python requests" optional = false -python-versions = ">=3.7,<4.0" +python-versions = ">=3.8" files = [ - {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, - {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, + {file = "requests_cache-1.2.0-py3-none-any.whl", hash = "sha256:490324301bf0cb924ff4e6324bd2613453e7e1f847353928b08adb0fdfb7f722"}, + {file = "requests_cache-1.2.0.tar.gz", hash = "sha256:db1c709ca343cc1cd5b6c8b1a5387298eceed02306a6040760db538c885e3838"}, ] [package.dependencies] @@ -826,15 +826,15 @@ url-normalize = ">=1.4" urllib3 = ">=1.25.5" [package.extras] -all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=6.0.1)", "redis (>=3)", "ujson (>=5.4)"] bson = ["bson (>=0.5)"] -docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.9)"] dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] json = ["ujson (>=5.4)"] mongodb = ["pymongo (>=3)"] redis = ["redis (>=3)"] security = ["itsdangerous (>=2.0)"] -yaml = ["pyyaml (>=5.4)"] +yaml = ["pyyaml (>=6.0.1)"] [[package]] name = "requests-mock" @@ -857,19 +857,19 @@ test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "tes [[package]] name = "setuptools" -version = "69.1.0" +version = "69.1.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, - {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, + {file = "setuptools-69.1.1-py3-none-any.whl", hash = "sha256:02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56"}, + {file = "setuptools-69.1.1.tar.gz", hash = "sha256:5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" @@ -920,13 +920,13 @@ six = "*" [[package]] name = "urllib3" -version = "2.2.0" +version = "2.2.1" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.8" files = [ - {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, - {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, ] [package.extras] @@ -1031,4 +1031,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9,<3.12" -content-hash = "da1dbc89f0a40d0a16baa47814c1e57b38c5afec44baa89789bc069fe9a7a7af" +content-hash = "bd7fa426ac72c522d45fd595a0f7275a9c2cb47acfb58ac8bb17c414f4137276" diff --git a/airbyte-integrations/connectors/source-recharge/pyproject.toml b/airbyte-integrations/connectors/source-recharge/pyproject.toml index 15ca7a7471c8..8c9177cff41a 100644 --- a/airbyte-integrations/connectors/source-recharge/pyproject.toml +++ b/airbyte-integrations/connectors/source-recharge/pyproject.toml @@ -17,7 +17,7 @@ include = "source_recharge" [tool.poetry.dependencies] python = "^3.9,<3.12" -airbyte-cdk = "==0.60.1" +airbyte-cdk = "^0.65.0" [tool.poetry.scripts] source-recharge = "source_recharge.run:run" diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py b/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py index 0bf213b881bf..7faf13405d2a 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py @@ -6,15 +6,15 @@ from dataclasses import dataclass from typing import Any, Mapping, Optional -from airbyte_cdk.sources.declarative.types import Record, StreamSlice from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor +from airbyte_cdk.sources.declarative.types import Record, StreamSlice @dataclass class RechargeDateTimeBasedCursor(DatetimeBasedCursor): """ Override for the default `DatetimeBasedCursor` to make self.close_slice() to produce `min` value instead of `max` value. - + This is the ONLY CHANGE MADE HERE, to make the SOURCE STATE proccessed correctly: The `min` value should be determined, in the first place, since we would skip the records, if they are updated manually, by the Customer, and the range in not AFTER the STATE value, but before. diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml b/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml index 8228265d8434..d777e3c3ade8 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml @@ -1,12 +1,6 @@ version: 0.65.0 definitions: - # API VERSION REFERENCE - deprecated_api_version: - x-recharge-version: "2021-01" - modern_api_version: - x-recharge-version: "2021-11" - # COMMON PARTS schema_loader: type: JsonFileSchemaLoader @@ -26,7 +20,7 @@ definitions: inject_into: type: RequestOption inject_into: header - field_name: X-Recharge-Access-Token + field_name: X-Recharge-Access-Token # PAGINATORS paginator_deprecated_api: @@ -73,22 +67,21 @@ definitions: error_handler: type: DefaultErrorHandler description: >- - We should retry the SUCCESSFULL request (200), with incomplete response.content - Case when: `data declared to receive` less than `actually received` + We should retry the SUCCESSFULL request (200), with incomplete response.content + Case when: `data declared to receive` bigger than `actually received` response_filters: - action: RETRY predicate: "{{ response.content|length > 0 and headers.get('Content-Length', 0)|int > response.content|length }}" - requester_deprecated_api: $ref: "#/definitions/requester_base" # for deprecated retriever we should use `2021-01` api version request_headers: - $ref: "#/definitions/deprecated_api_version" + x-recharge-version: "2021-01" requester_modern_api: $ref: "#/definitions/requester_base" # for modern retriever we should use >= `2021-11` api version request_headers: - $ref: "#/definitions/modern_api_version" + x-recharge-version: "2021-11" # RETRIEVER FOR `DEPRECATED API` retriever_api_deprecated: @@ -134,7 +127,7 @@ definitions: # $ref: "#/definitions/requester_modern_api" # request_parameters: # owner_resource: "{{ stream_partition['owner_resource'] }}" - + # BASE STREAMS # FULL-REFRESH base_stream: @@ -157,12 +150,12 @@ definitions: class_name: source_recharge.components.datetime_based_cursor.RechargeDateTimeBasedCursor cursor_field: "updated_at" cursor_datetime_formats: - - '%Y-%m-%dT%H:%M:%S%z' - datetime_format: '%Y-%m-%dT%H:%M:%S%z' + - "%Y-%m-%dT%H:%M:%S%z" + datetime_format: "%Y-%m-%dT%H:%M:%S%z" start_datetime: type: MinMaxDatetime - datetime: '{{ config[''start_date''] }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption field_name: "updated_at_min" @@ -186,7 +179,7 @@ definitions: $ref: "#/definitions/retriever_metafields" $parameters: name: "metafields" - data_path: "metafields" + data_path: "metafields" # PRODUCTS products_stream: description: >- @@ -211,8 +204,8 @@ definitions: field_path: [] primary_key: ["shop", "store"] $parameters: - name: "shop" - + name: "shop" + # INCREMENTAL STREAMS # ADDRESSES addresses_stream: diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/source.py b/airbyte-integrations/connectors/source-recharge/source_recharge/source.py index f2f5d0c873c3..be0b9d43509d 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/source.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/source.py @@ -4,8 +4,9 @@ from typing import Any, List, Mapping -from airbyte_cdk.sources.streams import Stream + from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource +from airbyte_cdk.sources.streams import Stream from source_recharge.streams import Orders, RechargeTokenAuthenticator """ From 4e892a2cff59110a87b767426c7abea08b071b1e Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Tue, 27 Feb 2024 14:02:11 +0200 Subject: [PATCH 07/33] extended schemas with missing properties --- .../source_recharge/schemas/addresses.json | 71 ++++++++- .../source_recharge/schemas/charges.json | 125 ++++++++++++++++ .../source_recharge/schemas/customers.json | 27 ++++ .../source_recharge/schemas/orders.json | 115 +++++++++++++++ .../source_recharge/schemas/products.json | 3 + .../schemas/subscriptions.json | 21 +++ .../source_recharge/streams.py | 136 ++++++++++++++++++ 7 files changed, 497 insertions(+), 1 deletion(-) create mode 100644 airbyte-integrations/connectors/source-recharge/source_recharge/streams.py diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/addresses.json b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/addresses.json index a8ec355493cd..489e7b128f4f 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/addresses.json +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/addresses.json @@ -49,7 +49,40 @@ "type": ["null", "string"] }, "shipping_lines_override": { - "type": ["null", "array"] + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "code": { + "type": ["null", "string"] + }, + "price": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + } + } + } + }, + "shipping_lines_conserved": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "code": { + "type": ["null", "string"] + }, + "price": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + } + } + } }, "updated_at": { "type": ["null", "string"], @@ -57,6 +90,42 @@ }, "zip": { "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "discounts": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + } + } + } + }, + "order_attributes": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "string"] + } + } + } + }, + "order_note": { + "type": ["null", "string"] + }, + "payment_method_id": { + "type": ["null", "integer"] + }, + "presentment_currency": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/charges.json b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/charges.json index 40faaeb50fe6..4e70968fd811 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/charges.json +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/charges.json @@ -35,6 +35,9 @@ "country": { "type": ["null", "string"] }, + "country_code": { + "type": ["null", "string"] + }, "customer_id": { "type": ["null", "integer"] }, @@ -72,6 +75,125 @@ } } }, + "charge_attempts": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "customer": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "email": { + "type": ["null", "string"] + }, + "external_customer_id": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "ecommerce": { + "type": ["null", "string"] + } + } + }, + "hash": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + } + } + }, + "discounts": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "code": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "integer"] + }, + "value_type": { + "type": ["null", "string"] + } + } + } + }, + "external_order_id": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "ecommerce": { + "type": ["null", "string"] + } + } + }, + "external_transaction_id": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "payment_processor": { + "type": ["null", "string"] + } + } + }, + "external_variant_id_not_found": { + "type": ["null", "boolean"] + }, + "external_variant_not_found": { + "type": ["null", "boolean"] + }, + "has_uncommitted_changes": { + "type": ["null", "boolean"] + }, + "last_charge_attempt": { + "type": ["null", "string"], + "format": "date-time" + }, + "merged_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "order_attributes": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + }, + "orders_count": { + "type": ["null", "integer"] + }, + "payment_processor": { + "type": ["null", "string"] + }, + "total_duties": { + "type": ["null", "string"] + }, + "total_weight_grams": { + "type": ["null", "integer"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "taxes_included": { + "type": ["null", "boolean"] + }, "client_details": { "type": ["null", "object"] }, @@ -166,6 +288,9 @@ "country": { "type": ["null", "string"] }, + "country_code": { + "type": ["null", "string"] + }, "customer_id": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/customers.json b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/customers.json index 9771a8c7cdb6..bb1a56d26b81 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/customers.json +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/customers.json @@ -94,6 +94,33 @@ }, "apply_credit_to_next_recurring_charge": { "type": ["null", "boolean"] + }, + "apply_credit_to_next_checkout_charge": { + "type": ["null", "boolean"] + }, + "external_customer_id": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "ecommerce": { + "type": ["null", "string"] + } + } + }, + "has_payment_method_in_dunning": { + "type": ["null", "boolean"] + }, + "phone": { + "type": ["null", "string"] + }, + "subscriptions_active_count": { + "type": ["null", "integer"] + }, + "subscriptions_total_count": { + "type": ["null", "integer"] + }, + "tax_exempt": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/orders.json b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/orders.json index 0112494af378..0bdec0fb8047 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/orders.json +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/orders.json @@ -5,6 +5,18 @@ "id": { "type": ["null", "integer"] }, + "accepts_marketing": { + "type": ["null", "boolean"] + }, + "send_email_welcome": { + "type": ["null", "boolean"] + }, + "verified_email": { + "type": ["null", "boolean"] + }, + "phone": { + "type": ["null", "string"] + }, "address_id": { "type": ["null", "integer"] }, @@ -35,6 +47,9 @@ "country": { "type": ["null", "string"] }, + "country_code": { + "type": ["null", "string"] + }, "customer_id": { "type": ["null", "integer"] }, @@ -184,6 +199,27 @@ "id": { "type": ["null", "integer"] }, + "accepts_marketing": { + "type": ["null", "boolean"] + }, + "send_email_welcome": { + "type": ["null", "boolean"] + }, + "verified_email": { + "type": ["null", "boolean"] + }, + "phone": { + "type": ["null", "string"] + }, + "external_customer_id": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "ecommerce": { + "type": ["null", "string"] + } + } + }, "billing_address1": { "type": ["null", "string"] }, @@ -293,9 +329,78 @@ "external_inventory_policy": { "type": ["null", "string"] }, + "external_product_id": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "ecommerce": { + "type": ["null", "string"] + } + } + }, + "external_variant_id": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "ecommerce": { + "type": ["null", "string"] + } + } + }, "grams": { "type": ["null", "number"] }, + "handle": { + "type": ["null", "string"] + }, + "purchase_item_id": { + "type": ["null", "integer"] + }, + "purchase_item_type": { + "type": ["null", "string"] + }, + "tax_due": { + "type": ["null", "string"] + }, + "taxable_amount": { + "type": ["null", "string"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "total_price": { + "type": ["null", "string"] + }, + "unit_price": { + "type": ["null", "string"] + }, + "unit_price_includes_tax": { + "type": ["null", "boolean"] + }, + "original_price": { + "type": ["null", "number"] + }, + "product_title": { + "type": ["null", "string"] + }, + "tax_lines": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "price": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + } + } + } + }, "images": { "type": ["null", "object"] }, @@ -377,6 +482,9 @@ "country": { "type": ["null", "string"] }, + "country_code": { + "type": ["null", "string"] + }, "customer_id": { "type": ["null", "integer"] }, @@ -468,6 +576,13 @@ "updated_at": { "type": ["null", "string"], "format": "date-time" + }, + "shipping_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "shopify_id": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/products.json b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/products.json index d0e10f87794c..3d76001d76ee 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/products.json +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/products.json @@ -5,6 +5,9 @@ "id": { "type": ["null", "integer"] }, + "product_id": { + "type": ["null", "integer"] + }, "charge_interval_frequency": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/subscriptions.json b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/subscriptions.json index 7556bb211549..40fca1f49e63 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/subscriptions.json +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/subscriptions.json @@ -40,6 +40,27 @@ "expire_after_specific_number_of_charges": { "type": ["null", "integer"] }, + "external_product_id": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "ecommerce": { + "type": ["null", "string"] + } + } + }, + "external_variant_id": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "ecommerce": { + "type": ["null", "string"] + } + } + }, + "presentment_currency": { + "type": ["null", "string"] + }, "has_queued_charges": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/streams.py b/airbyte-integrations/connectors/source-recharge/source_recharge/streams.py new file mode 100644 index 000000000000..0b7d2999e4e0 --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/streams.py @@ -0,0 +1,136 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import ABC +from enum import Enum +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional + +import pendulum +import requests +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer + + +class ApiVersion(Enum): + DEPRECATED = "2021-01" + MODERN = "2021-11" + + +class RechargeTokenAuthenticator(TokenAuthenticator): + def get_auth_header(self) -> Mapping[str, Any]: + return {"X-Recharge-Access-Token": self._token} + + +class Orders(HttpStream, ABC): + """ + Orders Stream: https://developer.rechargepayments.com/v1-shopify?python#list-orders + Notes: + Using `2021-01` the: `email`, `first_name`, `last_name` columns are not available, + because these are not present in `2021-11` as DEPRECATED fields. + """ + + primary_key: str = "id" + url_base: str = "https://api.rechargeapps.com/" + cursor_field: str = "updated_at" + page_size: int = 250 + page_num: int = 1 + period_in_days: int = 30 # Slice data request for 1 month + raise_on_http_errors: bool = True + state_checkpoint_interval: int = 250 + + # registering the default schema transformation + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + + def __init__(self, config, **kwargs) -> None: + super().__init__(**kwargs) + self._start_date = config["start_date"] + self.api_version = ApiVersion.DEPRECATED if config.get("use_orders_deprecated_api") else ApiVersion.MODERN + + @property + def data_path(self) -> str: + return self.name + + def request_headers(self, **kwargs) -> Mapping[str, Any]: + return {"x-recharge-version": self.api_version.value} + + def path(self, **kwargs) -> str: + return self.name + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + next_page_token = None + if self.api_version == ApiVersion.MODERN: + cursor = response.json().get("next_cursor") + if cursor: + next_page_token = {"cursor": cursor} + else: + stream_data = self.get_stream_data(response.json()) + if len(stream_data) == self.page_size: + self.page_num += 1 + next_page_token = {"page": self.page_num} + return next_page_token + + def _update_params_with_min_max_date_range( + self, + params: MutableMapping[str, Any], + stream_slice: Optional[Mapping[str, Any]] = None, + ) -> MutableMapping[str, Any]: + params.update( + { + "updated_at_min": (stream_slice or {}).get("start_date"), + "updated_at_max": (stream_slice or {}).get("end_date"), + } + ) + return params + + def request_params( + self, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs + ) -> MutableMapping[str, Any]: + params = {"limit": self.page_size} + if self.api_version == ApiVersion.MODERN: + # if a cursor value is passed, only limit can be passed with it! + if next_page_token: + params.update(next_page_token) + else: + params = self._update_params_with_min_max_date_range(params, stream_slice) + return params + else: + params = self._update_params_with_min_max_date_range(params, stream_slice) + if next_page_token: + params.update(next_page_token) + return params + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response_data = response.json() + stream_data = self.get_stream_data(response_data) + yield from stream_data + + def get_stream_data(self, response_data: Any) -> List[dict]: + if self.data_path: + return response_data.get(self.data_path, []) + else: + return [response_data] + + def should_retry(self, response: requests.Response) -> bool: + content_length = int(response.headers.get("Content-Length", 0)) + incomplete_data_response = response.status_code == 200 and content_length > len(response.content) + if incomplete_data_response: + return True + return super().should_retry(response) + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + start_date_value = (stream_state or {}).get(self.cursor_field, self._start_date) if self.cursor_field else self._start_date + now = pendulum.now() + # dates are inclusive, so we add 1 second so that time periods do not overlap + start_date = pendulum.parse(start_date_value).add(seconds=1) + while start_date <= now: + end_date = start_date.add(days=self.period_in_days) + yield {"start_date": start_date.strftime("%Y-%m-%d %H:%M:%S"), "end_date": end_date.strftime("%Y-%m-%d %H:%M:%S")} + start_date = end_date.add(seconds=1) + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + latest_benchmark = latest_record[self.cursor_field] + if current_stream_state.get(self.cursor_field): + return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])} + return {self.cursor_field: latest_benchmark} From fd3248835b7b6a0a297e272876e7e77feb275efc Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Tue, 27 Feb 2024 14:39:39 +0200 Subject: [PATCH 08/33] updated expected records, after schema changes --- .../acceptance-test-config.yml | 12 +------- .../integration_tests/expected_records.jsonl | 28 +++++++++---------- .../expected_records_orders_modern_api.jsonl | 28 +++++++++---------- 3 files changed, 29 insertions(+), 39 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml index 085c9b7cfc55..0ecc8110e3b6 100644 --- a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml @@ -15,13 +15,8 @@ acceptance_tests: bypass_reason: "updated after login" - name: store/updated_at bypass_reason: "updated after login" - timeout_seconds: 7200 expect_records: path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes - fail_on_extra_columns: false - config_path: secrets/config_order_modern_api.json empty_streams: - name: collections @@ -36,13 +31,8 @@ acceptance_tests: bypass_reason: "updated after login" - name: store/updated_at bypass_reason: "updated after login" - timeout_seconds: 7200 expect_records: path: "integration_tests/expected_records_orders_modern_api.jsonl" - extra_fields: no - exact_order: no - extra_records: yes - fail_on_extra_columns: false connection: tests: - config_path: secrets/config.json @@ -52,7 +42,7 @@ acceptance_tests: discovery: tests: - backward_compatibility_tests_config: - disable_for_version: 1.1.2 + disable_for_version: 1.1.5 config_path: secrets/config.json full_refresh: tests: diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl index 491a25b9836a..fde8d4ba0dd2 100644 --- a/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl @@ -1,14 +1,14 @@ -{"stream": "addresses", "data": {"id": 69105381, "customer_id": 64817252, "payment_method_id": 12482012, "address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "created_at": "2021-05-12T12:04:06+00:00", "discounts": [], "first_name": "Jane", "last_name": "Doe", "order_attributes": [], "order_note": null, "phone": "1234567890", "presentment_currency": "USD", "province": "California", "shipping_lines_conserved": [], "shipping_lines_override": [], "updated_at": "2023-01-16T09:59:09+00:00", "zip": "94118"}, "emitted_at": 1706644129288} -{"stream": "addresses", "data": {"id": 69282975, "customer_id": 64962974, "payment_method_id": 12482030, "address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "created_at": "2021-05-13T09:46:46+00:00", "discounts": [], "first_name": "Kelly", "last_name": "Kozakevich", "order_attributes": [], "order_note": null, "phone": "+16145550188", "presentment_currency": "USD", "province": "Illinois", "shipping_lines_conserved": [], "shipping_lines_override": [], "updated_at": "2023-05-13T04:07:34+00:00", "zip": "60510"}, "emitted_at": 1706644130026} -{"stream": "charges", "data": {"id": 386976088, "address_id": 69105381, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Karina", "last_name": "Kuznetsova", "phone": null, "province": "California", "zip": "94118"}, "charge_attempts": 6, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2021-05-12T12:04:07+00:00", "currency": "USD", "customer": {"id": 64817252, "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "hash": "23dee52d73734a81"}, "discounts": [], "error": "None\r\n [May 12, 12:06AM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 13, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 19, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 25, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 31, 4:09PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [Jun 06, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']", "error_type": "CLOSED_MAX_RETRIES_REACHED", "external_order_id": {"ecommerce": null}, "external_transaction_id": {"payment_processor": null}, "external_variant_not_found": null, "has_uncommitted_changes": false, "last_charge_attempt": "2022-06-06T20:10:19+00:00", "line_items": [{"purchase_item_id": 153224593, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684722131115"}, "grams": 0, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T1", "tax_due": "0.00", "tax_lines": [], "taxable": true, "taxable_amount": "24.30", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "S / Black"}], "merged_at": null, "note": null, "order_attributes": [], "orders_count": 0, "payment_processor": "shopify_payments", "processed_at": null, "retry_date": "2022-06-12T04:00:00+00:00", "scheduled_at": "2022-05-12", "shipping_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Jane", "last_name": "Doe", "phone": "1234567890", "province": "California", "zip": "94118"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "error", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": true, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-01-16T18:08:54+00:00"}, "emitted_at": 1706644132446} -{"stream": "charges", "data": {"id": 817715206, "address_id": 69282975, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2023-05-13T04:07:34+00:00", "currency": "USD", "customer": {"id": 64962974, "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "hash": "f99bd4a6877257af"}, "discounts": [], "error": null, "error_type": null, "external_order_id": {"ecommerce": null}, "external_transaction_id": {"payment_processor": null}, "has_uncommitted_changes": false, "line_items": [{"purchase_item_id": 153601366, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "grams": 0, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T3", "tax_due": "0.00", "tax_lines": [], "taxable": true, "taxable_amount": "24.30", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "L / City Green"}], "merged_at": null, "note": null, "order_attributes": [], "orders_count": 0, "payment_processor": "shopify_payments", "processed_at": null, "retry_date": null, "scheduled_at": "2024-05-12", "shipping_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "queued", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": true, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-05-13T04:07:47+00:00"}, "emitted_at": 1706644133275} -{"stream": "charges", "data": {"id": 580825303, "address_id": 69282975, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2022-05-13T04:07:39+00:00", "currency": "USD", "customer": {"id": 64962974, "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "hash": "f99bd4a6877257af"}, "discounts": [], "error": null, "error_type": null, "external_order_id": {"ecommerce": "5006149877931"}, "external_transaction_id": {"payment_processor": "43114102955"}, "has_uncommitted_changes": false, "line_items": [{"purchase_item_id": 153601366, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "grams": null, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T3", "tax_due": "0.00", "tax_lines": [], "taxable": false, "taxable_amount": "0.00", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "L / City Green"}], "merged_at": null, "note": null, "order_attributes": [], "orders_count": 1, "payment_processor": "shopify_payments", "processed_at": "2023-05-13T04:07:33+00:00", "retry_date": null, "scheduled_at": "2023-05-13", "shipping_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "success", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": false, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-05-13T04:16:51+00:00"}, "emitted_at": 1706644133278} -{"stream": "customers", "data": {"id": 64817252, "analytics_data": {"utm_params": []}, "apply_credit_to_next_checkout_charge": false, "apply_credit_to_next_recurring_charge": false, "created_at": "2021-05-12T12:04:06+00:00", "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "first_charge_processed_at": "2021-05-12T16:03:59+00:00", "first_name": "Karina", "has_payment_method_in_dunning": false, "has_valid_payment_method": true, "hash": "23dee52d73734a81", "last_name": "Kuznetsova", "phone": null, "subscriptions_active_count": 0, "subscriptions_total_count": 1, "tax_exempt": false, "updated_at": "2023-01-16T18:08:45+00:00"}, "emitted_at": 1706644139386} -{"stream": "customers", "data": {"id": 64962974, "analytics_data": {"utm_params": []}, "apply_credit_to_next_checkout_charge": false, "apply_credit_to_next_recurring_charge": false, "created_at": "2021-05-13T09:46:44+00:00", "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "first_charge_processed_at": "2021-05-13T13:46:39+00:00", "first_name": "Kelly", "has_payment_method_in_dunning": false, "has_valid_payment_method": true, "hash": "f99bd4a6877257af", "last_name": "Kozakevich", "phone": "+16145550188", "subscriptions_active_count": 1, "subscriptions_total_count": 1, "tax_exempt": false, "updated_at": "2023-05-13T04:16:36+00:00"}, "emitted_at": 1706644140190} -{"stream": "metafields", "data": {"id": 3627108, "owner_id": "64962974", "created_at": "2023-04-10T07:10:45", "description": "customer_phone_number", "key": "phone_number", "namespace": "personal_info", "owner_resource": "customer", "updated_at": "2023-04-10T07:10:45", "value": "3103103101", "value_type": "integer"}, "emitted_at": 1706644151126} -{"stream": "orders", "data": {"address_id": 69282975, "address_is_active": 1, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country": "United States", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "browser_ip": null, "charge_id": 580825303, "charge_status": "SUCCESS", "created_at": "2023-05-13T00:07:28", "currency": "USD", "customer": {"accepts_marketing": true, "email": "kozakevich_k@example.com", "first_name": "Kelly", "last_name": "Kozakevich", "phone": null, "send_email_welcome": false, "verified_email": true}, "customer_id": 64962974, "discount_codes": null, "email": "kozakevich_k@example.com", "error": null, "first_name": "Kelly", "hash": "f99bd4a6877257af", "id": 534919106, "is_prepaid": 0, "last_name": "Kozakevich", "line_items": [{"external_inventory_policy": "decrement_obeying_policy", "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": 24.3, "price": 24.3, "product_title": "Airbit Box Corner Short sleeve t-shirt", "properties": [], "quantity": 1, "shopify_product_id": "6642695864491", "shopify_variant_id": "39684723835051", "sku": "T3", "subscription_id": 153601366, "tax_lines": [], "title": "Airbit Box Corner Short sleeve t-shirt", "variant_title": "L / City Green"}], "note": null, "note_attributes": [], "payment_processor": "shopify_payments", "processed_at": "2023-05-13T00:07:33", "scheduled_at": "2023-05-13T00:00:00", "shipped_date": "2023-05-13T00:07:33", "shipping_address": {"address1": "1921 W Wilson St", "address2": "", "city": "Batavia", "company": null, "country": "United States", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_date": "2023-05-13T00:00:00", "shipping_lines": [{"code": "Economy", "price": "4.90", "source": "shopify", "title": "Economy"}], "shopify_cart_token": null, "shopify_customer_id": "5213433266347", "shopify_id": "5006149877931", "shopify_order_id": "5006149877931", "shopify_order_number": 1016, "status": "SUCCESS", "subtotal_price": 24.3, "tags": "Subscription, Subscription Recurring Order", "tax_lines": [], "total_discounts": 0.0, "total_duties": "0.0", "total_line_items_price": 24.3, "total_price": 29.2, "total_refunds": null, "total_tax": "0.0", "total_weight": 0, "transaction_id": "43114102955", "type": "RECURRING", "updated_at": "2023-05-13T00:16:51"}, "emitted_at": 1706644162075} -{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T08:08:28", "discount_amount": 5.0, "discount_type": "percentage", "handle": "airbit-box-corner-short-sleeve-t-shirt", "id": 1853649, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "product_id": 6642695864491, "shopify_product_id": 6642695864491, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "Airbit Box Corner Short sleeve t-shirt", "updated_at": "2021-05-13T08:08:28"}, "emitted_at": 1706644170248} -{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T07:27:34", "discount_amount": 5.0, "discount_type": "percentage", "handle": "i-make-beats-wool-blend-snapback", "id": 1853639, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_small.jpg"}, "product_id": 6644278001835, "shopify_product_id": 6644278001835, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "I Make Beats Wool Blend Snapback", "updated_at": "2021-05-13T07:27:34"}, "emitted_at": 1706644170251} -{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T08:20:10", "discount_amount": 0.0, "discount_type": "percentage", "handle": "new-mug", "id": 1853655, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_small.jpg"}, "product_id": 6688261701803, "shopify_product_id": 6688261701803, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "NEW!!! MUG", "updated_at": "2021-05-13T08:20:10"}, "emitted_at": 1706644170252} -{"stream": "shop", "data": {"shop": {"allow_customers_to_skip_delivery": 1, "checkout_logo_url": null, "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Thu, 13 Jul 2023 15:26:57 GMT"}, "store": {"checkout_logo_url": null, "checkout_platform": "shopify", "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Thu, 13 Jul 2023 15:26:57 GMT"}}, "emitted_at": 1706644179022} -{"stream": "subscriptions", "data": {"id": 153601366, "address_id": 69282975, "customer_id": 64962974, "analytics_data": {"utm_params": []}, "cancellation_reason": null, "cancellation_reason_comments": null, "cancelled_at": null, "charge_interval_frequency": "365", "created_at": "2021-05-13T09:46:47+00:00", "expire_after_specific_number_of_charges": null, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "has_queued_charges": 1, "is_prepaid": false, "is_skippable": true, "is_swappable": false, "max_retries_reached": 0, "next_charge_scheduled_at": "2024-05-12", "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency": "365", "order_interval_unit": "day", "presentment_currency": "USD", "price": 24.3, "product_title": "Airbit Box Corner Short sleeve t-shirt", "properties": [], "quantity": 1, "sku": null, "sku_override": false, "status": "active", "updated_at": "2023-05-13T04:07:32+00:00", "variant_title": "L / City Green"}, "emitted_at": 1706644181724} +{"stream": "addresses", "data": {"id": 69282975, "customer_id": 64962974, "payment_method_id": 12482030, "address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "created_at": "2021-05-13T09:46:46+00:00", "discounts": [], "first_name": "Kelly", "last_name": "Kozakevich", "order_attributes": [], "order_note": null, "phone": "+16145550188", "presentment_currency": "USD", "province": "Illinois", "shipping_lines_conserved": [], "shipping_lines_override": [], "updated_at": "2023-05-13T04:07:34+00:00", "zip": "60510"}, "emitted_at": 1709035723343} +{"stream": "addresses", "data": {"id": 69105381, "customer_id": 64817252, "payment_method_id": 12482012, "address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "created_at": "2021-05-12T12:04:06+00:00", "discounts": [], "first_name": "Jane", "last_name": "Doe", "order_attributes": [], "order_note": null, "phone": "1234567890", "presentment_currency": "USD", "province": "California", "shipping_lines_conserved": [], "shipping_lines_override": [], "updated_at": "2023-01-16T09:59:09+00:00", "zip": "94118"}, "emitted_at": 1709035723348} +{"stream": "charges", "data": {"id": 817715206, "address_id": 69282975, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2023-05-13T04:07:34+00:00", "currency": "USD", "customer": {"id": 64962974, "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "hash": "f99bd4a6877257af"}, "discounts": [], "error": null, "error_type": null, "external_order_id": {"ecommerce": null}, "external_transaction_id": {"payment_processor": null}, "has_uncommitted_changes": false, "line_items": [{"purchase_item_id": 153601366, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "grams": 0, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T3", "tax_due": "0.00", "tax_lines": [], "taxable": true, "taxable_amount": "24.30", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "L / City Green"}], "merged_at": null, "note": null, "order_attributes": [], "orders_count": 0, "payment_processor": "shopify_payments", "processed_at": null, "retry_date": null, "scheduled_at": "2024-05-12", "shipping_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "queued", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": true, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-05-13T04:07:47+00:00"}, "emitted_at": 1709035724071} +{"stream": "charges", "data": {"id": 580825303, "address_id": 69282975, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2022-05-13T04:07:39+00:00", "currency": "USD", "customer": {"id": 64962974, "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "hash": "f99bd4a6877257af"}, "discounts": [], "error": null, "error_type": null, "external_order_id": {"ecommerce": "5006149877931"}, "external_transaction_id": {"payment_processor": "43114102955"}, "has_uncommitted_changes": false, "line_items": [{"purchase_item_id": 153601366, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "grams": null, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T3", "tax_due": "0.00", "tax_lines": [], "taxable": false, "taxable_amount": "0.00", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "L / City Green"}], "merged_at": null, "note": null, "order_attributes": [], "orders_count": 1, "payment_processor": "shopify_payments", "processed_at": "2023-05-13T04:07:33+00:00", "retry_date": null, "scheduled_at": "2023-05-13", "shipping_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "success", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": false, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-05-13T04:16:51+00:00"}, "emitted_at": 1709035724078} +{"stream": "charges", "data": {"id": 386976088, "address_id": 69105381, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Karina", "last_name": "Kuznetsova", "phone": null, "province": "California", "zip": "94118"}, "charge_attempts": 6, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2021-05-12T12:04:07+00:00", "currency": "USD", "customer": {"id": 64817252, "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "hash": "23dee52d73734a81"}, "discounts": [], "error": "None\r\n [May 12, 12:06AM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 13, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 19, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 25, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 31, 4:09PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [Jun 06, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']", "error_type": "CLOSED_MAX_RETRIES_REACHED", "external_order_id": {"ecommerce": null}, "external_transaction_id": {"payment_processor": null}, "external_variant_not_found": null, "has_uncommitted_changes": false, "last_charge_attempt": "2022-06-06T20:10:19+00:00", "line_items": [{"purchase_item_id": 153224593, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684722131115"}, "grams": 0, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T1", "tax_due": "0.00", "tax_lines": [], "taxable": true, "taxable_amount": "24.30", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "S / Black"}], "merged_at": null, "note": null, "order_attributes": [], "orders_count": 0, "payment_processor": "shopify_payments", "processed_at": null, "retry_date": "2022-06-12T04:00:00+00:00", "scheduled_at": "2022-05-12", "shipping_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Jane", "last_name": "Doe", "phone": "1234567890", "province": "California", "zip": "94118"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "error", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": true, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-01-16T18:08:54+00:00"}, "emitted_at": 1709035724083} +{"stream": "customers", "data": {"id": 64962974, "analytics_data": {"utm_params": []}, "apply_credit_to_next_checkout_charge": false, "apply_credit_to_next_recurring_charge": false, "created_at": "2021-05-13T09:46:44+00:00", "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "first_charge_processed_at": "2021-05-13T13:46:39+00:00", "first_name": "Kelly", "has_payment_method_in_dunning": false, "has_valid_payment_method": true, "hash": "f99bd4a6877257af", "last_name": "Kozakevich", "phone": "+16145550188", "subscriptions_active_count": 1, "subscriptions_total_count": 1, "tax_exempt": false, "updated_at": "2023-05-13T04:16:36+00:00"}, "emitted_at": 1709035725565} +{"stream": "customers", "data": {"id": 64817252, "analytics_data": {"utm_params": []}, "apply_credit_to_next_checkout_charge": false, "apply_credit_to_next_recurring_charge": false, "created_at": "2021-05-12T12:04:06+00:00", "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "first_charge_processed_at": "2021-05-12T16:03:59+00:00", "first_name": "Karina", "has_payment_method_in_dunning": false, "has_valid_payment_method": true, "hash": "23dee52d73734a81", "last_name": "Kuznetsova", "phone": null, "subscriptions_active_count": 0, "subscriptions_total_count": 1, "tax_exempt": false, "updated_at": "2023-01-16T18:08:45+00:00"}, "emitted_at": 1709035725569} +{"stream": "metafields", "data": {"id": 3627108, "owner_id": "64962974", "created_at": "2023-04-10T07:10:45", "description": "customer_phone_number", "key": "phone_number", "namespace": "personal_info", "owner_resource": "customer", "updated_at": "2023-04-10T07:10:45", "value": "3103103101", "value_type": "integer"}, "emitted_at": 1709035727500} +{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T08:08:28", "discount_amount": 5.0, "discount_type": "percentage", "handle": "airbit-box-corner-short-sleeve-t-shirt", "id": 1853649, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "product_id": 6642695864491, "shopify_product_id": 6642695864491, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "Airbit Box Corner Short sleeve t-shirt", "updated_at": "2021-05-13T08:08:28"}, "emitted_at": 1709035729322} +{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T07:27:34", "discount_amount": 5.0, "discount_type": "percentage", "handle": "i-make-beats-wool-blend-snapback", "id": 1853639, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_small.jpg"}, "product_id": 6644278001835, "shopify_product_id": 6644278001835, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "I Make Beats Wool Blend Snapback", "updated_at": "2021-05-13T07:27:34"}, "emitted_at": 1709035729325} +{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T08:20:10", "discount_amount": 0.0, "discount_type": "percentage", "handle": "new-mug", "id": 1853655, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_small.jpg"}, "product_id": 6688261701803, "shopify_product_id": 6688261701803, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "NEW!!! MUG", "updated_at": "2021-05-13T08:20:10"}, "emitted_at": 1709035729328} +{"stream": "shop", "data": {"shop": {"allow_customers_to_skip_delivery": 1, "checkout_logo_url": null, "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Tue, 30 Jan 2024 18:56:54 GMT"}, "store": {"checkout_logo_url": null, "checkout_platform": "shopify", "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Tue, 30 Jan 2024 18:56:54 GMT"}}, "emitted_at": 1709035729971} +{"stream": "subscriptions", "data": {"id": 153601366, "address_id": 69282975, "customer_id": 64962974, "analytics_data": {"utm_params": []}, "cancellation_reason": null, "cancellation_reason_comments": null, "cancelled_at": null, "charge_interval_frequency": "365", "created_at": "2021-05-13T09:46:47+00:00", "expire_after_specific_number_of_charges": null, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "has_queued_charges": 1, "is_prepaid": false, "is_skippable": true, "is_swappable": false, "max_retries_reached": 0, "next_charge_scheduled_at": "2024-05-12", "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency": "365", "order_interval_unit": "day", "presentment_currency": "USD", "price": 24.3, "product_title": "Airbit Box Corner Short sleeve t-shirt", "properties": [], "quantity": 1, "sku": null, "sku_override": false, "status": "active", "updated_at": "2023-05-13T04:07:32+00:00", "variant_title": "L / City Green"}, "emitted_at": 1709035730656} +{"stream": "orders", "data": {"address_id": 69282975, "address_is_active": 1, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country": "United States", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "browser_ip": null, "charge_id": 580825303, "charge_status": "SUCCESS", "created_at": "2023-05-13T00:07:28", "currency": "USD", "customer": {"accepts_marketing": true, "email": "kozakevich_k@example.com", "first_name": "Kelly", "last_name": "Kozakevich", "phone": null, "send_email_welcome": false, "verified_email": true}, "customer_id": 64962974, "discount_codes": null, "email": "kozakevich_k@example.com", "error": null, "first_name": "Kelly", "hash": "f99bd4a6877257af", "id": 534919106, "is_prepaid": 0, "last_name": "Kozakevich", "line_items": [{"external_inventory_policy": "decrement_obeying_policy", "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": 24.3, "price": 24.3, "product_title": "Airbit Box Corner Short sleeve t-shirt", "properties": [], "quantity": 1, "shopify_product_id": "6642695864491", "shopify_variant_id": "39684723835051", "sku": "T3", "subscription_id": 153601366, "tax_lines": [], "title": "Airbit Box Corner Short sleeve t-shirt", "variant_title": "L / City Green"}], "note": null, "note_attributes": [], "payment_processor": "shopify_payments", "processed_at": "2023-05-13T00:07:33", "scheduled_at": "2023-05-13T00:00:00", "shipped_date": "2023-05-13T00:07:33", "shipping_address": {"address1": "1921 W Wilson St", "address2": "", "city": "Batavia", "company": null, "country": "United States", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_date": "2023-05-13T00:00:00", "shipping_lines": [{"code": "Economy", "price": "4.90", "source": "shopify", "title": "Economy"}], "shopify_cart_token": null, "shopify_customer_id": "5213433266347", "shopify_id": "5006149877931", "shopify_order_id": "5006149877931", "shopify_order_number": 1016, "status": "SUCCESS", "subtotal_price": 24.3, "tags": "Subscription, Subscription Recurring Order", "tax_lines": [], "total_discounts": 0.0, "total_duties": "0.0", "total_line_items_price": 24.3, "total_price": 29.2, "total_refunds": null, "total_tax": "0.0", "total_weight": 0, "transaction_id": "43114102955", "type": "RECURRING", "updated_at": "2023-05-13T00:16:51"}, "emitted_at": 1709035732348} diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records_orders_modern_api.jsonl b/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records_orders_modern_api.jsonl index 9962d32adf33..93fc0f46961a 100644 --- a/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records_orders_modern_api.jsonl +++ b/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records_orders_modern_api.jsonl @@ -1,14 +1,14 @@ -{"stream": "addresses", "data": {"id": 69105381, "customer_id": 64817252, "payment_method_id": 12482012, "address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "created_at": "2021-05-12T12:04:06+00:00", "discounts": [], "first_name": "Jane", "last_name": "Doe", "order_attributes": [], "order_note": null, "phone": "1234567890", "presentment_currency": "USD", "province": "California", "shipping_lines_conserved": [], "shipping_lines_override": [], "updated_at": "2023-01-16T09:59:09+00:00", "zip": "94118"}, "emitted_at": 1706644270838} -{"stream": "addresses", "data": {"id": 69282975, "customer_id": 64962974, "payment_method_id": 12482030, "address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "created_at": "2021-05-13T09:46:46+00:00", "discounts": [], "first_name": "Kelly", "last_name": "Kozakevich", "order_attributes": [], "order_note": null, "phone": "+16145550188", "presentment_currency": "USD", "province": "Illinois", "shipping_lines_conserved": [], "shipping_lines_override": [], "updated_at": "2023-05-13T04:07:34+00:00", "zip": "60510"}, "emitted_at": 1706644271610} -{"stream": "charges", "data": {"id": 386976088, "address_id": 69105381, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Karina", "last_name": "Kuznetsova", "phone": null, "province": "California", "zip": "94118"}, "charge_attempts": 6, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2021-05-12T12:04:07+00:00", "currency": "USD", "customer": {"id": 64817252, "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "hash": "23dee52d73734a81"}, "discounts": [], "error": "None\r\n [May 12, 12:06AM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 13, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 19, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 25, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 31, 4:09PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [Jun 06, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']", "error_type": "CLOSED_MAX_RETRIES_REACHED", "external_order_id": {"ecommerce": null}, "external_transaction_id": {"payment_processor": null}, "external_variant_not_found": null, "has_uncommitted_changes": false, "last_charge_attempt": "2022-06-06T20:10:19+00:00", "line_items": [{"purchase_item_id": 153224593, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684722131115"}, "grams": 0, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T1", "tax_due": "0.00", "tax_lines": [], "taxable": true, "taxable_amount": "24.30", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "S / Black"}], "merged_at": null, "note": null, "order_attributes": [], "orders_count": 0, "payment_processor": "shopify_payments", "processed_at": null, "retry_date": "2022-06-12T04:00:00+00:00", "scheduled_at": "2022-05-12", "shipping_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Jane", "last_name": "Doe", "phone": "1234567890", "province": "California", "zip": "94118"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "error", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": true, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-01-16T18:08:54+00:00"}, "emitted_at": 1706644274123} -{"stream": "charges", "data": {"id": 817715206, "address_id": 69282975, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2023-05-13T04:07:34+00:00", "currency": "USD", "customer": {"id": 64962974, "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "hash": "f99bd4a6877257af"}, "discounts": [], "error": null, "error_type": null, "external_order_id": {"ecommerce": null}, "external_transaction_id": {"payment_processor": null}, "has_uncommitted_changes": false, "line_items": [{"purchase_item_id": 153601366, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "grams": 0, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T3", "tax_due": "0.00", "tax_lines": [], "taxable": true, "taxable_amount": "24.30", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "L / City Green"}], "merged_at": null, "note": null, "order_attributes": [], "orders_count": 0, "payment_processor": "shopify_payments", "processed_at": null, "retry_date": null, "scheduled_at": "2024-05-12", "shipping_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "queued", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": true, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-05-13T04:07:47+00:00"}, "emitted_at": 1706644274939} -{"stream": "charges", "data": {"id": 580825303, "address_id": 69282975, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2022-05-13T04:07:39+00:00", "currency": "USD", "customer": {"id": 64962974, "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "hash": "f99bd4a6877257af"}, "discounts": [], "error": null, "error_type": null, "external_order_id": {"ecommerce": "5006149877931"}, "external_transaction_id": {"payment_processor": "43114102955"}, "has_uncommitted_changes": false, "line_items": [{"purchase_item_id": 153601366, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "grams": null, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T3", "tax_due": "0.00", "tax_lines": [], "taxable": false, "taxable_amount": "0.00", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "L / City Green"}], "merged_at": null, "note": null, "order_attributes": [], "orders_count": 1, "payment_processor": "shopify_payments", "processed_at": "2023-05-13T04:07:33+00:00", "retry_date": null, "scheduled_at": "2023-05-13", "shipping_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "success", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": false, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-05-13T04:16:51+00:00"}, "emitted_at": 1706644274942} -{"stream": "customers", "data": {"id": 64817252, "analytics_data": {"utm_params": []}, "apply_credit_to_next_checkout_charge": false, "apply_credit_to_next_recurring_charge": false, "created_at": "2021-05-12T12:04:06+00:00", "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "first_charge_processed_at": "2021-05-12T16:03:59+00:00", "first_name": "Karina", "has_payment_method_in_dunning": false, "has_valid_payment_method": true, "hash": "23dee52d73734a81", "last_name": "Kuznetsova", "phone": null, "subscriptions_active_count": 0, "subscriptions_total_count": 1, "tax_exempt": false, "updated_at": "2023-01-16T18:08:45+00:00"}, "emitted_at": 1706644280530} -{"stream": "customers", "data": {"id": 64962974, "analytics_data": {"utm_params": []}, "apply_credit_to_next_checkout_charge": false, "apply_credit_to_next_recurring_charge": false, "created_at": "2021-05-13T09:46:44+00:00", "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "first_charge_processed_at": "2021-05-13T13:46:39+00:00", "first_name": "Kelly", "has_payment_method_in_dunning": false, "has_valid_payment_method": true, "hash": "f99bd4a6877257af", "last_name": "Kozakevich", "phone": "+16145550188", "subscriptions_active_count": 1, "subscriptions_total_count": 1, "tax_exempt": false, "updated_at": "2023-05-13T04:16:36+00:00"}, "emitted_at": 1706644281267} -{"stream": "metafields", "data": {"id": 3627108, "owner_id": "64962974", "created_at": "2023-04-10T07:10:45", "description": "customer_phone_number", "key": "phone_number", "namespace": "personal_info", "owner_resource": "customer", "updated_at": "2023-04-10T07:10:45", "value": "3103103101", "value_type": "integer"}, "emitted_at": 1706644292270} -{"stream": "orders", "data": {"id": 534919106, "address_id": 69282975, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "charge": {"id": 580825303, "external_transaction_id": {"payment_processor": "43114102955"}, "payment_processor_name": "shopify_payments", "status": "success"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2023-05-13T04:07:28+00:00", "currency": "USD", "customer": {"id": 64962974, "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "hash": "f99bd4a6877257af"}, "discounts": [], "error": null, "external_cart_token": null, "external_order_id": {"ecommerce": "5006149877931"}, "external_order_name": {"ecommerce": "#1016"}, "external_order_number": {"ecommerce": "1016"}, "is_prepaid": 0, "line_items": [{"purchase_item_id": 153601366, "external_inventory_policy": "decrement_obeying_policy", "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "grams": null, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T3", "tax_due": "0.00", "tax_lines": [], "taxable": false, "taxable_amount": "0.00", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "L / City Green"}], "note": null, "order_attributes": [], "processed_at": "2023-05-13T04:07:33+00:00", "scheduled_at": "2023-05-13T04:00:00+00:00", "shipping_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "source": "shopify", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "success", "subtotal_price": 24.3, "tags": "Subscription, Subscription Recurring Order", "tax_lines": [], "taxable": false, "total_discounts": 0.0, "total_duties": "0.00", "total_line_items_price": 24.3, "total_price": 29.2, "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-05-13T04:16:51+00:00"}, "emitted_at": 1706644303256} -{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T08:08:28", "discount_amount": 5.0, "discount_type": "percentage", "handle": "airbit-box-corner-short-sleeve-t-shirt", "id": 1853649, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "product_id": 6642695864491, "shopify_product_id": 6642695864491, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "Airbit Box Corner Short sleeve t-shirt", "updated_at": "2021-05-13T08:08:28"}, "emitted_at": 1706644311039} -{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T07:27:34", "discount_amount": 5.0, "discount_type": "percentage", "handle": "i-make-beats-wool-blend-snapback", "id": 1853639, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_small.jpg"}, "product_id": 6644278001835, "shopify_product_id": 6644278001835, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "I Make Beats Wool Blend Snapback", "updated_at": "2021-05-13T07:27:34"}, "emitted_at": 1706644311045} -{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T08:20:10", "discount_amount": 0.0, "discount_type": "percentage", "handle": "new-mug", "id": 1853655, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_small.jpg"}, "product_id": 6688261701803, "shopify_product_id": 6688261701803, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "NEW!!! MUG", "updated_at": "2021-05-13T08:20:10"}, "emitted_at": 1706644311046} -{"stream": "shop", "data": {"shop": {"allow_customers_to_skip_delivery": 1, "checkout_logo_url": null, "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Thu, 13 Jul 2023 15:26:57 GMT"}, "store": {"checkout_logo_url": null, "checkout_platform": "shopify", "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Thu, 13 Jul 2023 15:26:57 GMT"}}, "emitted_at": 1706644319680} -{"stream": "subscriptions", "data": {"id": 153601366, "address_id": 69282975, "customer_id": 64962974, "analytics_data": {"utm_params": []}, "cancellation_reason": null, "cancellation_reason_comments": null, "cancelled_at": null, "charge_interval_frequency": "365", "created_at": "2021-05-13T09:46:47+00:00", "expire_after_specific_number_of_charges": null, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "has_queued_charges": 1, "is_prepaid": false, "is_skippable": true, "is_swappable": false, "max_retries_reached": 0, "next_charge_scheduled_at": "2024-05-12", "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency": "365", "order_interval_unit": "day", "presentment_currency": "USD", "price": 24.3, "product_title": "Airbit Box Corner Short sleeve t-shirt", "properties": [], "quantity": 1, "sku": null, "sku_override": false, "status": "active", "updated_at": "2023-05-13T04:07:32+00:00", "variant_title": "L / City Green"}, "emitted_at": 1706644322400} +{"stream": "addresses", "data": {"id": 69282975, "customer_id": 64962974, "payment_method_id": 12482030, "address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "created_at": "2021-05-13T09:46:46+00:00", "discounts": [], "first_name": "Kelly", "last_name": "Kozakevich", "order_attributes": [], "order_note": null, "phone": "+16145550188", "presentment_currency": "USD", "province": "Illinois", "shipping_lines_conserved": [], "shipping_lines_override": [], "updated_at": "2023-05-13T04:07:34+00:00", "zip": "60510"}, "emitted_at": 1709035647334} +{"stream": "addresses", "data": {"id": 69105381, "customer_id": 64817252, "payment_method_id": 12482012, "address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "created_at": "2021-05-12T12:04:06+00:00", "discounts": [], "first_name": "Jane", "last_name": "Doe", "order_attributes": [], "order_note": null, "phone": "1234567890", "presentment_currency": "USD", "province": "California", "shipping_lines_conserved": [], "shipping_lines_override": [], "updated_at": "2023-01-16T09:59:09+00:00", "zip": "94118"}, "emitted_at": 1709035647340} +{"stream": "charges", "data": {"id": 817715206, "address_id": 69282975, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2023-05-13T04:07:34+00:00", "currency": "USD", "customer": {"id": 64962974, "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "hash": "f99bd4a6877257af"}, "discounts": [], "error": null, "error_type": null, "external_order_id": {"ecommerce": null}, "external_transaction_id": {"payment_processor": null}, "has_uncommitted_changes": false, "line_items": [{"purchase_item_id": 153601366, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "grams": 0, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T3", "tax_due": "0.00", "tax_lines": [], "taxable": true, "taxable_amount": "24.30", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "L / City Green"}], "merged_at": null, "note": null, "order_attributes": [], "orders_count": 0, "payment_processor": "shopify_payments", "processed_at": null, "retry_date": null, "scheduled_at": "2024-05-12", "shipping_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "queued", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": true, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-05-13T04:07:47+00:00"}, "emitted_at": 1709035648060} +{"stream": "charges", "data": {"id": 580825303, "address_id": 69282975, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2022-05-13T04:07:39+00:00", "currency": "USD", "customer": {"id": 64962974, "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "hash": "f99bd4a6877257af"}, "discounts": [], "error": null, "error_type": null, "external_order_id": {"ecommerce": "5006149877931"}, "external_transaction_id": {"payment_processor": "43114102955"}, "has_uncommitted_changes": false, "line_items": [{"purchase_item_id": 153601366, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "grams": null, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T3", "tax_due": "0.00", "tax_lines": [], "taxable": false, "taxable_amount": "0.00", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "L / City Green"}], "merged_at": null, "note": null, "order_attributes": [], "orders_count": 1, "payment_processor": "shopify_payments", "processed_at": "2023-05-13T04:07:33+00:00", "retry_date": null, "scheduled_at": "2023-05-13", "shipping_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "success", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": false, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-05-13T04:16:51+00:00"}, "emitted_at": 1709035648066} +{"stream": "charges", "data": {"id": 386976088, "address_id": 69105381, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Karina", "last_name": "Kuznetsova", "phone": null, "province": "California", "zip": "94118"}, "charge_attempts": 6, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2021-05-12T12:04:07+00:00", "currency": "USD", "customer": {"id": 64817252, "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "hash": "23dee52d73734a81"}, "discounts": [], "error": "None\r\n [May 12, 12:06AM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 13, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 19, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 25, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 31, 4:09PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [Jun 06, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']", "error_type": "CLOSED_MAX_RETRIES_REACHED", "external_order_id": {"ecommerce": null}, "external_transaction_id": {"payment_processor": null}, "external_variant_not_found": null, "has_uncommitted_changes": false, "last_charge_attempt": "2022-06-06T20:10:19+00:00", "line_items": [{"purchase_item_id": 153224593, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684722131115"}, "grams": 0, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T1", "tax_due": "0.00", "tax_lines": [], "taxable": true, "taxable_amount": "24.30", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "S / Black"}], "merged_at": null, "note": null, "order_attributes": [], "orders_count": 0, "payment_processor": "shopify_payments", "processed_at": null, "retry_date": "2022-06-12T04:00:00+00:00", "scheduled_at": "2022-05-12", "shipping_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Jane", "last_name": "Doe", "phone": "1234567890", "province": "California", "zip": "94118"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "error", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": true, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-01-16T18:08:54+00:00"}, "emitted_at": 1709035648071} +{"stream": "customers", "data": {"id": 64962974, "analytics_data": {"utm_params": []}, "apply_credit_to_next_checkout_charge": false, "apply_credit_to_next_recurring_charge": false, "created_at": "2021-05-13T09:46:44+00:00", "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "first_charge_processed_at": "2021-05-13T13:46:39+00:00", "first_name": "Kelly", "has_payment_method_in_dunning": false, "has_valid_payment_method": true, "hash": "f99bd4a6877257af", "last_name": "Kozakevich", "phone": "+16145550188", "subscriptions_active_count": 1, "subscriptions_total_count": 1, "tax_exempt": false, "updated_at": "2023-05-13T04:16:36+00:00"}, "emitted_at": 1709035649348} +{"stream": "customers", "data": {"id": 64817252, "analytics_data": {"utm_params": []}, "apply_credit_to_next_checkout_charge": false, "apply_credit_to_next_recurring_charge": false, "created_at": "2021-05-12T12:04:06+00:00", "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "first_charge_processed_at": "2021-05-12T16:03:59+00:00", "first_name": "Karina", "has_payment_method_in_dunning": false, "has_valid_payment_method": true, "hash": "23dee52d73734a81", "last_name": "Kuznetsova", "phone": null, "subscriptions_active_count": 0, "subscriptions_total_count": 1, "tax_exempt": false, "updated_at": "2023-01-16T18:08:45+00:00"}, "emitted_at": 1709035649352} +{"stream": "metafields", "data": {"id": 3627108, "owner_id": "64962974", "created_at": "2023-04-10T07:10:45", "description": "customer_phone_number", "key": "phone_number", "namespace": "personal_info", "owner_resource": "customer", "updated_at": "2023-04-10T07:10:45", "value": "3103103101", "value_type": "integer"}, "emitted_at": 1709035651342} +{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T08:08:28", "discount_amount": 5.0, "discount_type": "percentage", "handle": "airbit-box-corner-short-sleeve-t-shirt", "id": 1853649, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "product_id": 6642695864491, "shopify_product_id": 6642695864491, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "Airbit Box Corner Short sleeve t-shirt", "updated_at": "2021-05-13T08:08:28"}, "emitted_at": 1709035653155} +{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T07:27:34", "discount_amount": 5.0, "discount_type": "percentage", "handle": "i-make-beats-wool-blend-snapback", "id": 1853639, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_small.jpg"}, "product_id": 6644278001835, "shopify_product_id": 6644278001835, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "I Make Beats Wool Blend Snapback", "updated_at": "2021-05-13T07:27:34"}, "emitted_at": 1709035653159} +{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T08:20:10", "discount_amount": 0.0, "discount_type": "percentage", "handle": "new-mug", "id": 1853655, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_small.jpg"}, "product_id": 6688261701803, "shopify_product_id": 6688261701803, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "NEW!!! MUG", "updated_at": "2021-05-13T08:20:10"}, "emitted_at": 1709035653161} +{"stream": "shop", "data": {"shop": {"allow_customers_to_skip_delivery": 1, "checkout_logo_url": null, "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Tue, 30 Jan 2024 18:56:54 GMT"}, "store": {"checkout_logo_url": null, "checkout_platform": "shopify", "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Tue, 30 Jan 2024 18:56:54 GMT"}}, "emitted_at": 1709035654067} +{"stream": "subscriptions", "data": {"id": 153601366, "address_id": 69282975, "customer_id": 64962974, "analytics_data": {"utm_params": []}, "cancellation_reason": null, "cancellation_reason_comments": null, "cancelled_at": null, "charge_interval_frequency": "365", "created_at": "2021-05-13T09:46:47+00:00", "expire_after_specific_number_of_charges": null, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "has_queued_charges": 1, "is_prepaid": false, "is_skippable": true, "is_swappable": false, "max_retries_reached": 0, "next_charge_scheduled_at": "2024-05-12", "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency": "365", "order_interval_unit": "day", "presentment_currency": "USD", "price": 24.3, "product_title": "Airbit Box Corner Short sleeve t-shirt", "properties": [], "quantity": 1, "sku": null, "sku_override": false, "status": "active", "updated_at": "2023-05-13T04:07:32+00:00", "variant_title": "L / City Green"}, "emitted_at": 1709035655558} +{"stream": "orders", "data": {"id": 534919106, "address_id": 69282975, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "charge": {"id": 580825303, "external_transaction_id": {"payment_processor": "43114102955"}, "payment_processor_name": "shopify_payments", "status": "success"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2023-05-13T04:07:28+00:00", "currency": "USD", "customer": {"id": 64962974, "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "hash": "f99bd4a6877257af"}, "discounts": [], "error": null, "external_cart_token": null, "external_order_id": {"ecommerce": "5006149877931"}, "external_order_name": {"ecommerce": "#1016"}, "external_order_number": {"ecommerce": "1016"}, "is_prepaid": 0, "line_items": [{"purchase_item_id": 153601366, "external_inventory_policy": "decrement_obeying_policy", "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "grams": null, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": 24.3, "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T3", "tax_due": "0.00", "tax_lines": [], "taxable": false, "taxable_amount": "0.00", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "L / City Green"}], "note": null, "order_attributes": [], "processed_at": "2023-05-13T04:07:33+00:00", "scheduled_at": "2023-05-13T04:00:00+00:00", "shipping_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "source": "shopify", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "success", "subtotal_price": 24.3, "tags": "Subscription, Subscription Recurring Order", "tax_lines": [], "taxable": false, "total_discounts": 0.0, "total_duties": "0.00", "total_line_items_price": 24.3, "total_price": 29.2, "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-05-13T04:16:51+00:00"}, "emitted_at": 1709035661452} From 6d6b48c2008aa18c7f72f3e2786684ff7bd8a0ab Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Tue, 27 Feb 2024 16:08:13 +0200 Subject: [PATCH 09/33] refactored old unit_tests up to the changes --- .../connectors/source-recharge/README.md | 2 +- .../integration_tests/configured_catalog.json | 13 + .../source-recharge/unit_tests/conftest.py | 19 + .../source-recharge/unit_tests/test_api.py | 364 ------------------ .../source-recharge/unit_tests/test_source.py | 49 +-- .../unit_tests/test_streams.py | 248 ++++++++++++ 6 files changed, 282 insertions(+), 413 deletions(-) create mode 100644 airbyte-integrations/connectors/source-recharge/unit_tests/conftest.py delete mode 100644 airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py create mode 100644 airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-recharge/README.md b/airbyte-integrations/connectors/source-recharge/README.md index 03ee5b5b276e..c09b9cde6f48 100644 --- a/airbyte-integrations/connectors/source-recharge/README.md +++ b/airbyte-integrations/connectors/source-recharge/README.md @@ -30,7 +30,7 @@ See `sample_files/sample_config.json` for a sample config file. poetry run source-recharge spec poetry run source-recharge check --config secrets/config.json poetry run source-recharge discover --config secrets/config.json -poetry run source-recharge read --config secrets/config.json --catalog sample_files/configured_catalog.json +poetry run source-recharge read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` ### Running unit tests diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-recharge/integration_tests/configured_catalog.json index b08c61a13137..6a716c1f8f36 100644 --- a/airbyte-integrations/connectors/source-recharge/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-recharge/integration_tests/configured_catalog.json @@ -62,6 +62,19 @@ "destination_sync_mode": "append", "cursor_field": ["updated_at"] }, + { + "stream": { + "name": "onetimes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, { "stream": { "name": "discounts", diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/conftest.py b/airbyte-integrations/connectors/source-recharge/unit_tests/conftest.py new file mode 100644 index 000000000000..e02797d00053 --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/conftest.py @@ -0,0 +1,19 @@ +from typing import Any, Mapping +from unittest.mock import patch +import pytest + + +@pytest.fixture(name="config") +def config() -> Mapping[str, Any]: + return { + "authenticator": None, + "access_token": "access_token", + "start_date": "2021-08-15T00:00:00Z", + } + + +@pytest.fixture(name="logger_mock") +def logger_mock_fixture() -> None: + return patch("source_recharge.source.AirbyteLogger") + + diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py b/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py deleted file mode 100644 index 3981d725c047..000000000000 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py +++ /dev/null @@ -1,364 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import patch - -import pytest -import requests -from source_recharge.api import ( - Addresses, - Charges, - Collections, - Customers, - Discounts, - Metafields, - Onetimes, - OrdersDeprecatedApi, - OrdersModernApi, - Products, - RechargeStreamDeprecatedAPI, - RechargeStreamModernAPI, - Shop, - Subscriptions, -) - - -# config -@pytest.fixture(name="config") -def config(): - return { - "authenticator": None, - "access_token": "access_token", - "start_date": "2021-08-15T00:00:00Z", - } - - -class TestCommon: - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (Addresses, "id"), - (Charges, "id"), - (Collections, "id"), - (Customers, "id"), - (Discounts, "id"), - (Metafields, "id"), - (Onetimes, "id"), - (OrdersDeprecatedApi, "id"), - (OrdersModernApi, "id"), - (Products, "id"), - (Shop, ["shop", "store"]), - (Subscriptions, "id"), - ], - ) - def test_primary_key(self, stream_cls, expected): - assert expected == stream_cls.primary_key - - @pytest.mark.parametrize( - "stream_cls, stream_type, expected", - [ - (Addresses, "incremental", "addresses"), - (Charges, "incremental", "charges"), - (Collections, "full-refresh", "collections"), - (Customers, "incremental", "customers"), - (Discounts, "incremental", "discounts"), - (Metafields, "full-refresh", "metafields"), - (Onetimes, "incremental", "onetimes"), - (OrdersDeprecatedApi, "incremental", "orders"), - (OrdersModernApi, "incremental", "orders"), - (Products, "full-refresh", "products"), - (Shop, "full-refresh", None), - (Subscriptions, "incremental", "subscriptions"), - ], - ) - def test_data_path(self, config, stream_cls, stream_type, expected): - if stream_type == "incremental": - result = stream_cls(config, authenticator=None).data_path - else: - result = stream_cls(config, authenticator=None).data_path - assert expected == result - - @pytest.mark.parametrize( - "stream_cls, stream_type, expected", - [ - (Addresses, "incremental", "addresses"), - (Charges, "incremental", "charges"), - (Collections, "full-refresh", "collections"), - (Customers, "incremental", "customers"), - (Discounts, "incremental", "discounts"), - (Metafields, "full-refresh", "metafields"), - (Onetimes, "incremental", "onetimes"), - (OrdersDeprecatedApi, "incremental", "orders"), - (OrdersModernApi, "incremental", "orders"), - (Products, "full-refresh", "products"), - (Shop, "full-refresh", "shop"), - (Subscriptions, "incremental", "subscriptions"), - ], - ) - def test_path(self, config, stream_cls, stream_type, expected): - if stream_type == "incremental": - result = stream_cls(config, authenticator=None).path() - else: - result = stream_cls(config, authenticator=None).path() - assert expected == result - - @pytest.mark.parametrize( - ("http_status", "headers", "should_retry"), - [ - (HTTPStatus.OK, {"Content-Length": 256}, True), - (HTTPStatus.BAD_REQUEST, {}, False), - (HTTPStatus.TOO_MANY_REQUESTS, {}, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, {}, True), - (HTTPStatus.FORBIDDEN, {}, False), - ], - ) - @pytest.mark.parametrize("stream_cls", (RechargeStreamDeprecatedAPI, RechargeStreamModernAPI)) - def test_should_retry(self, config, http_status, headers, should_retry, stream_cls): - response = requests.Response() - response.status_code = http_status - response._content = b"" - response.headers = headers - stream = stream_cls(config, authenticator=None) - assert stream.should_retry(response) == should_retry - - -class TestFullRefreshStreams: - def generate_records(self, stream_name, count): - if not stream_name: - return {f"record_{1}": f"test_{1}"} - result = [] - for i in range(0, count): - result.append({f"record_{i}": f"test_{i}"}) - return {stream_name: result} - - @pytest.mark.parametrize( - "stream_cls, cursor_response, expected", - [ - (Collections, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), - (Metafields, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), - (OrdersModernApi, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), - (Products, {}, {"page": 2}), - (Shop, {}, None), - (OrdersDeprecatedApi, {}, {"page": 2}), - ], - ) - def test_next_page_token(self, config, stream_cls, cursor_response, requests_mock, expected): - stream = stream_cls(config, authenticator=None) - stream.limit = 2 - url = f"{stream.url_base}{stream.path()}" - response = {**cursor_response, **self.generate_records(stream.data_path, 2)} - requests_mock.get(url, json=response) - response = requests.get(url) - assert stream.next_page_token(response) == expected - - @pytest.mark.parametrize( - "stream_cls, next_page_token, stream_state, stream_slice, expected", - [ - ( - Collections, - None, - {}, - {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, - {"limit": 250, "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, - ), - (Metafields, {"cursor": "12353"}, {"updated_at": "2030-01-01"}, {}, {"limit": 250, "owner_resource": None, "cursor": "12353"}), - ( - Products, - None, - {}, - {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, - {"limit": 250, "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, - ), - (Shop, None, {}, {}, {}), - ], - ) - def test_request_params(self, config, stream_cls, next_page_token, stream_state, stream_slice, expected): - stream = stream_cls(config, authenticator=None) - result = stream.request_params(stream_state, stream_slice, next_page_token) - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, data, expected", - [ - (Collections, [{"test": 123}], [{"test": 123}]), - (Metafields, [{"test2": 234}], [{"test2": 234}]), - (Products, [{"test3": 345}], [{"test3": 345}]), - (Shop, {"test4": 456}, [{"test4": 456}]), - ], - ) - def test_parse_response(self, config, stream_cls, data, requests_mock, expected): - stream = stream_cls(config, authenticator=None) - url = f"{stream.url_base}{stream.path()}" - data = {stream.data_path: data} if stream.data_path else data - requests_mock.get(url, json=data) - response = requests.get(url) - assert list(stream.parse_response(response)) == expected - - @pytest.mark.parametrize( - "stream_cls, data, expected", - [ - (Collections, [{"test": 123}], [{"test": 123}]), - (Metafields, [{"test2": 234}], [{"test2": 234}]), - (Products, [{"test3": 345}], [{"test3": 345}]), - (Shop, {"test4": 456}, [{"test4": 456}]), - ], - ) - def get_stream_data(self, config, stream_cls, data, requests_mock, expected): - stream = stream_cls(config, authenticator=None) - url = f"{stream.url_base}{stream.path()}" - data = {stream.data_path: data} if stream.data_path else data - requests_mock.get(url, json=data) - response = requests.get(url) - assert list(stream.parse_response(response)) == expected - - @pytest.mark.parametrize("owner_resource, expected", [({"customer": {"id": 123}}, {"customer": {"id": 123}})]) - def test_metafields_read_records(self, config, owner_resource, expected): - with patch.object(Metafields, "read_records", return_value=owner_resource): - result = Metafields(config).read_records(stream_slice={"owner_resource": owner_resource}) - assert result == expected - - -class TestIncrementalStreams: - def generate_records(self, stream_name, count): - result = [] - for i in range(0, count): - result.append({f"record_{i}": f"test_{i}"}) - return {stream_name: result} - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (Addresses, "updated_at"), - (Charges, "updated_at"), - (Customers, "updated_at"), - (Discounts, "updated_at"), - (Onetimes, "updated_at"), - (OrdersDeprecatedApi, "updated_at"), - (OrdersModernApi, "updated_at"), - (Subscriptions, "updated_at"), - ], - ) - def test_cursor_field(self, config, stream_cls, expected): - stream = stream_cls(config, authenticator=None) - result = stream.cursor_field - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, cursor_response, expected", - [ - (Addresses, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), - (Charges, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), - (Customers, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), - (Discounts, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), - (Onetimes, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), - (OrdersDeprecatedApi, {}, {"page": 2}), - (OrdersModernApi, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), - (Subscriptions, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), - ], - ) - def test_next_page_token(self, config, stream_cls, cursor_response, requests_mock, expected): - stream = stream_cls(config, authenticator=None) - stream.limit = 2 - url = f"{stream.url_base}{stream.path()}" - response = {**cursor_response, **self.generate_records(stream.data_path, 2)} - requests_mock.get(url, json=response) - response = requests.get(url) - assert stream.next_page_token(response) == expected - - @pytest.mark.parametrize( - "stream_cls, next_page_token, stream_state, stream_slice, expected", - [ - ( - Addresses, - None, - {}, - {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, - {"limit": 250, "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, - ), - ( - Charges, - {"cursor": "123"}, - {"updated_at": "2030-01-01"}, - {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, - {"limit": 250, "cursor": "123"}, - ), - ( - Customers, - None, - {}, - {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, - {"limit": 250, "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, - ), - ( - Discounts, - None, - {}, - {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, - {"limit": 250, "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, - ), - ( - Onetimes, - {"cursor": "123"}, - {"updated_at": "2030-01-01"}, - {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, - {"limit": 250, "cursor": "123"}, - ), - ( - OrdersDeprecatedApi, - None, - {}, - {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, - {"limit": 250, "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, - ), - ( - OrdersModernApi, - None, - {}, - {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, - {"limit": 250, "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, - ), - ( - Subscriptions, - None, - {}, - {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, - {"limit": 250, "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, - ), - ], - ) - def test_request_params(self, config, stream_cls, next_page_token, stream_state, stream_slice, expected): - stream = stream_cls(config, authenticator=None) - result = stream.request_params(stream_state, stream_slice, next_page_token) - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, current_state, latest_record, expected", - [ - (Addresses, {}, {"updated_at": 2}, {"updated_at": 2}), - (Charges, {"updated_at": 2}, {"updated_at": 3}, {"updated_at": 3}), - (Customers, {"updated_at": 3}, {"updated_at": 4}, {"updated_at": 4}), - (Discounts, {}, {"updated_at": 2}, {"updated_at": 2}), - (Onetimes, {}, {"updated_at": 2}, {"updated_at": 2}), - (OrdersDeprecatedApi, {"updated_at": 5}, {"updated_at": 5}, {"updated_at": 5}), - (OrdersModernApi, {"updated_at": 5}, {"updated_at": 5}, {"updated_at": 5}), - (Subscriptions, {"updated_at": 6}, {"updated_at": 7}, {"updated_at": 7}), - ], - ) - def test_get_updated_state(self, config, stream_cls, current_state, latest_record, expected): - stream = stream_cls(config, authenticator=None) - result = stream.get_updated_state(current_state, latest_record) - assert result == expected - - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (Addresses, {'start_date': '2021-08-15 00:00:01', 'end_date': '2021-09-14 00:00:01'}), - ], - ) - def test_stream_slices(self, config, stream_cls, expected): - stream = stream_cls(config, authenticator=None) - result = list(stream.stream_slices(sync_mode=None, cursor_field=stream.cursor_field, stream_state=None)) - assert result[0] == expected diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/test_source.py b/airbyte-integrations/connectors/source-recharge/unit_tests/test_source.py index 17b8e92123f0..1a4e69e73850 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/test_source.py @@ -3,54 +3,7 @@ # -from unittest.mock import patch - -import pytest -from requests.exceptions import HTTPError -from source_recharge.api import Shop -from source_recharge.source import RechargeTokenAuthenticator, SourceRecharge - - -# config -@pytest.fixture(name="config") -def config(): - return { - "authenticator": None, - "access_token": "access_token", - "start_date": "2021-08-15T00:00:00Z", - } - - -# logger -@pytest.fixture(name="logger_mock") -def logger_mock_fixture(): - return patch("source_recharge.source.AirbyteLogger") - - -def test_get_auth_header(config): - expected = {"X-Recharge-Access-Token": config.get("access_token")} - actual = RechargeTokenAuthenticator(token=config["access_token"]).get_auth_header() - assert actual == expected - - -@pytest.mark.parametrize( - "patch, expected", - [ - ( - patch.object(Shop, "read_records", return_value=[{"shop": {"id": 123}}]), - (True, None), - ), - ( - patch.object(Shop, "read_records", side_effect=HTTPError(403)), - (False, "Unable to connect to Recharge API with the provided credentials - HTTPError(403)"), - ), - ], - ids=["success", "fail"], -) -def test_check_connection(logger_mock, config, patch, expected): - with patch: - result = SourceRecharge().check_connection(logger_mock, config=config) - assert result == expected +from source_recharge.source import SourceRecharge def test_streams(config): diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py new file mode 100644 index 000000000000..216be45a3a3c --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py @@ -0,0 +1,248 @@ + +import pytest +from typing import Any, Mapping, MutableMapping +from source_recharge.source import RechargeTokenAuthenticator, Orders +import requests +from http import HTTPStatus + + +def use_orders_deprecated_api_config( + config: Mapping[str, Any] = None, + use_deprecated_api: bool = False, +) -> MutableMapping[str, Any]: + test_config = config + if use_deprecated_api: + test_config["use_orders_deprecated_api"] = use_deprecated_api + return test_config + +def test_get_auth_header(config): + expected = {"X-Recharge-Access-Token": config.get("access_token")} + actual = RechargeTokenAuthenticator(token=config["access_token"]).get_auth_header() + assert actual == expected + + +class TestCommon: + @pytest.mark.parametrize( + "stream_cls, expected", + [ + (Orders, "id"), + ], + ) + def test_primary_key(self, stream_cls, expected): + assert expected == stream_cls.primary_key + + @pytest.mark.parametrize( + "stream_cls, stream_type, expected", + [ + (Orders, "incremental", "orders"), + ], + ) + def test_data_path(self, config, stream_cls, stream_type, expected): + if stream_type == "incremental": + result = stream_cls(config, authenticator=None).data_path + else: + result = stream_cls(config, authenticator=None).data_path + assert expected == result + + @pytest.mark.parametrize( + "stream_cls, stream_type, expected", + [ + (Orders, "incremental", "orders"), + ], + ) + def test_path(self, config, stream_cls, stream_type, expected): + if stream_type == "incremental": + result = stream_cls(config, authenticator=None).path() + else: + result = stream_cls(config, authenticator=None).path() + assert expected == result + + @pytest.mark.parametrize( + ("http_status", "headers", "should_retry"), + [ + (HTTPStatus.OK, {"Content-Length": 256}, True), + (HTTPStatus.BAD_REQUEST, {}, False), + (HTTPStatus.TOO_MANY_REQUESTS, {}, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, {}, True), + (HTTPStatus.FORBIDDEN, {}, False), + ], + ) + def test_should_retry(self, config, http_status, headers, should_retry): + response = requests.Response() + response.status_code = http_status + response._content = b"" + response.headers = headers + stream = Orders(config, authenticator=None) + assert stream.should_retry(response) == should_retry + + +class TestFullRefreshStreams: + def generate_records(self, stream_name, count): + if not stream_name: + return {f"record_{1}": f"test_{1}"} + result = [] + for i in range(0, count): + result.append({f"record_{i}": f"test_{i}"}) + return {stream_name: result} + + @pytest.mark.parametrize( + "stream_cls, use_deprecated_api, cursor_response, expected", + [ + (Orders, True, {}, {"page": 2}), + (Orders, False, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), + ], + ) + def test_next_page_token(self, config, use_deprecated_api, stream_cls, cursor_response, requests_mock, expected): + test_config = use_orders_deprecated_api_config(config, use_deprecated_api) + stream = stream_cls(test_config, authenticator=None) + stream.page_size = 2 + url = f"{stream.url_base}{stream.path()}" + response = {**cursor_response, **self.generate_records(stream.data_path, 2)} + requests_mock.get(url, json=response) + response = requests.get(url) + assert stream.next_page_token(response) == expected + + @pytest.mark.parametrize( + "stream_cls, use_deprecated_api, next_page_token, stream_slice, expected", + [ + ( + Orders, + True, + None, + {"start_date":"2023-01-01 00:00:01","end_date":"2023-01-31 00:00:01"}, + {'limit': 250, 'updated_at_min': '2023-01-01 00:00:01', 'updated_at_max': '2023-01-31 00:00:01'}, + ), + ( + Orders, + False, + None, + {"start_date":"2023-01-01 00:00:01","end_date":"2023-01-31 00:00:01"}, + {'limit': 250, 'updated_at_min': '2023-01-01 00:00:01', 'updated_at_max': '2023-01-31 00:00:01'}, + ), + ], + ) + def test_request_params(self, config, stream_cls, use_deprecated_api, next_page_token, stream_slice, expected): + test_config = use_orders_deprecated_api_config(config, use_deprecated_api) + stream = stream_cls(test_config, authenticator=None) + result = stream.request_params(stream_slice, next_page_token) + assert result == expected + + @pytest.mark.parametrize( + "stream_cls, use_deprecated_api, data, expected", + [ + (Orders, True, [{"test": 123}], [{"test": 123}]), + (Orders, False, [{"test": 123}], [{"test": 123}]), + ], + ) + def test_parse_response(self, config, stream_cls, use_deprecated_api, data, requests_mock, expected): + test_config = use_orders_deprecated_api_config(config, use_deprecated_api) + stream = stream_cls(test_config, authenticator=None) + url = f"{stream.url_base}{stream.path()}" + data = {stream.data_path: data} if stream.data_path else data + requests_mock.get(url, json=data) + response = requests.get(url) + assert list(stream.parse_response(response)) == expected + + @pytest.mark.parametrize( + "stream_cls, use_deprecated_api, data, expected", + [ + (Orders, True, [{"test": 123}], [{"test": 123}]), + (Orders, False, [{"test": 123}], [{"test": 123}]), + ], + ) + def get_stream_data(self, config, stream_cls, use_deprecated_api, data, requests_mock, expected): + test_config = use_orders_deprecated_api_config(config, use_deprecated_api) + stream = stream_cls(test_config, authenticator=None) + url = f"{stream.url_base}{stream.path()}" + data = {stream.data_path: data} if stream.data_path else data + requests_mock.get(url, json=data) + response = requests.get(url) + assert list(stream.parse_response(response)) == expected + + +class TestIncrementalStreams: + def generate_records(self, stream_name, count): + result = [] + for i in range(0, count): + result.append({f"record_{i}": f"test_{i}"}) + return {stream_name: result} + + @pytest.mark.parametrize( + "stream_cls, use_deprecated_api, expected", + [ + (Orders, True, "updated_at"), + (Orders, False, "updated_at"), + ], + ) + def test_cursor_field(self, config, stream_cls, use_deprecated_api, expected): + test_config = use_orders_deprecated_api_config(config, use_deprecated_api) + stream = stream_cls(test_config, authenticator=None) + result = stream.cursor_field + assert result == expected + + @pytest.mark.parametrize( + "stream_cls, use_deprecated_api, cursor_response, expected", + [ + (Orders, True, {}, {"page": 2}), + (Orders, False, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), + ], + ) + def test_next_page_token(self, config, stream_cls, use_deprecated_api, cursor_response, requests_mock, expected): + test_config = use_orders_deprecated_api_config(config, use_deprecated_api) + stream = stream_cls(test_config, authenticator=None) + stream.page_size = 2 + url = f"{stream.url_base}{stream.path()}" + response = {**cursor_response, **self.generate_records(stream.data_path, 2)} + requests_mock.get(url, json=response) + response = requests.get(url) + assert stream.next_page_token(response) == expected + + @pytest.mark.parametrize( + "stream_cls, use_deprecated_api, next_page_token, stream_slice, expected", + [ + ( + Orders, + True, + None, + {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, + {'limit': 250, 'updated_at_min': '2020-01-01T00:00:00Z', 'updated_at_max': '2020-02-01T00:00:00Z'}, + ), + ( + Orders, + False, + None, + {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, + {'limit': 250, 'updated_at_min': '2020-01-01T00:00:00Z', 'updated_at_max': '2020-02-01T00:00:00Z'}, + ), + ], + ) + def test_request_params(self, config, stream_cls, use_deprecated_api, next_page_token, stream_slice, expected): + test_config = use_orders_deprecated_api_config(config, use_deprecated_api) + stream = stream_cls(test_config, authenticator=None) + result = stream.request_params(stream_slice, next_page_token) + assert result == expected + + + @pytest.mark.parametrize( + "stream_cls, use_deprecated_api, current_state, latest_record, expected", + [ + (Orders, True, {"updated_at": 5}, {"updated_at": 5}, {"updated_at": 5}), + (Orders, False, {"updated_at": 5}, {"updated_at": 5}, {"updated_at": 5}), + ], + ) + def test_get_updated_state(self, config, stream_cls,use_deprecated_api, current_state, latest_record, expected): + test_config = use_orders_deprecated_api_config(config, use_deprecated_api) + stream = stream_cls(test_config, authenticator=None) + result = stream.get_updated_state(current_state, latest_record) + assert result == expected + + @pytest.mark.parametrize( + "stream_cls, expected", + [ + (Orders, {'start_date': '2021-08-15 00:00:01', 'end_date': '2021-09-14 00:00:01'}), + ], + ) + def test_stream_slices(self, config, stream_cls, expected): + stream = stream_cls(config, authenticator=None) + result = list(stream.stream_slices(sync_mode=None, cursor_field=stream.cursor_field, stream_state=None)) + assert result[0] == expected \ No newline at end of file From 055f94b05e6e7f54f7b4a21e095dd140485dd464 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 28 Feb 2024 12:03:07 +0200 Subject: [PATCH 10/33] added freezegun to dependencies --- .../connectors/source-recharge/poetry.lock | 18 +++- .../connectors/source-recharge/pyproject.toml | 1 + .../source-recharge/unit_tests/conftest.py | 1 + .../source-recharge/unit_tests/test_source.py | 11 --- .../unit_tests/test_streams.py | 82 +++++++++++-------- 5 files changed, 64 insertions(+), 49 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-recharge/unit_tests/test_source.py diff --git a/airbyte-integrations/connectors/source-recharge/poetry.lock b/airbyte-integrations/connectors/source-recharge/poetry.lock index bfd053afdb5b..b749358041e5 100644 --- a/airbyte-integrations/connectors/source-recharge/poetry.lock +++ b/airbyte-integrations/connectors/source-recharge/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "airbyte-cdk" @@ -301,6 +301,20 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "freezegun" +version = "1.4.0" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.7" +files = [ + {file = "freezegun-1.4.0-py3-none-any.whl", hash = "sha256:55e0fc3c84ebf0a96a5aa23ff8b53d70246479e9a68863f1fcac5a3e52f19dd6"}, + {file = "freezegun-1.4.0.tar.gz", hash = "sha256:10939b0ba0ff5adaecf3b06a5c2f73071d9678e507c5eaedb23c761d56ac774b"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + [[package]] name = "genson" version = "1.2.2" @@ -1031,4 +1045,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9,<3.12" -content-hash = "bd7fa426ac72c522d45fd595a0f7275a9c2cb47acfb58ac8bb17c414f4137276" +content-hash = "6cc8daf1f69b1beb2f355e9d58a27076059aaccdf716b1db1c6399e5bdd7c207" diff --git a/airbyte-integrations/connectors/source-recharge/pyproject.toml b/airbyte-integrations/connectors/source-recharge/pyproject.toml index 8c9177cff41a..c2f2326a9bb0 100644 --- a/airbyte-integrations/connectors/source-recharge/pyproject.toml +++ b/airbyte-integrations/connectors/source-recharge/pyproject.toml @@ -18,6 +18,7 @@ include = "source_recharge" [tool.poetry.dependencies] python = "^3.9,<3.12" airbyte-cdk = "^0.65.0" +freezegun = "^1.4.0" [tool.poetry.scripts] source-recharge = "source_recharge.run:run" diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/conftest.py b/airbyte-integrations/connectors/source-recharge/unit_tests/conftest.py index e02797d00053..52626e56566c 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/conftest.py @@ -1,5 +1,6 @@ from typing import Any, Mapping from unittest.mock import patch + import pytest diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/test_source.py b/airbyte-integrations/connectors/source-recharge/unit_tests/test_source.py deleted file mode 100644 index 1a4e69e73850..000000000000 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/test_source.py +++ /dev/null @@ -1,11 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from source_recharge.source import SourceRecharge - - -def test_streams(config): - streams = SourceRecharge().streams(config) - assert len(streams) == 11 diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py index 216be45a3a3c..8c64f31668c8 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py @@ -1,13 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from http import HTTPStatus +from typing import Any, List, Mapping, MutableMapping import pytest -from typing import Any, Mapping, MutableMapping -from source_recharge.source import RechargeTokenAuthenticator, Orders import requests -from http import HTTPStatus +from source_recharge.source import Orders, RechargeTokenAuthenticator, SourceRecharge def use_orders_deprecated_api_config( - config: Mapping[str, Any] = None, + config: Mapping[str, Any] = None, use_deprecated_api: bool = False, ) -> MutableMapping[str, Any]: test_config = config @@ -15,12 +20,18 @@ def use_orders_deprecated_api_config( test_config["use_orders_deprecated_api"] = use_deprecated_api return test_config -def test_get_auth_header(config): + +def test_get_auth_header(config) -> None: expected = {"X-Recharge-Access-Token": config.get("access_token")} actual = RechargeTokenAuthenticator(token=config["access_token"]).get_auth_header() assert actual == expected +def test_streams(config) -> None: + streams = SourceRecharge().streams(config) + assert len(streams) == 11 + + class TestCommon: @pytest.mark.parametrize( "stream_cls, expected", @@ -28,7 +39,7 @@ class TestCommon: (Orders, "id"), ], ) - def test_primary_key(self, stream_cls, expected): + def test_primary_key(self, stream_cls, expected) -> None: assert expected == stream_cls.primary_key @pytest.mark.parametrize( @@ -37,7 +48,7 @@ def test_primary_key(self, stream_cls, expected): (Orders, "incremental", "orders"), ], ) - def test_data_path(self, config, stream_cls, stream_type, expected): + def test_data_path(self, config, stream_cls, stream_type, expected) -> None: if stream_type == "incremental": result = stream_cls(config, authenticator=None).data_path else: @@ -50,7 +61,7 @@ def test_data_path(self, config, stream_cls, stream_type, expected): (Orders, "incremental", "orders"), ], ) - def test_path(self, config, stream_cls, stream_type, expected): + def test_path(self, config, stream_cls, stream_type, expected) -> None: if stream_type == "incremental": result = stream_cls(config, authenticator=None).path() else: @@ -67,7 +78,7 @@ def test_path(self, config, stream_cls, stream_type, expected): (HTTPStatus.FORBIDDEN, {}, False), ], ) - def test_should_retry(self, config, http_status, headers, should_retry): + def test_should_retry(self, config, http_status, headers, should_retry) -> None: response = requests.Response() response.status_code = http_status response._content = b"" @@ -77,7 +88,7 @@ def test_should_retry(self, config, http_status, headers, should_retry): class TestFullRefreshStreams: - def generate_records(self, stream_name, count): + def generate_records(self, stream_name, count) -> Mapping[str, List[Mapping[str, Any]]] | Mapping[str, Any]: if not stream_name: return {f"record_{1}": f"test_{1}"} result = [] @@ -92,7 +103,7 @@ def generate_records(self, stream_name, count): (Orders, False, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), ], ) - def test_next_page_token(self, config, use_deprecated_api, stream_cls, cursor_response, requests_mock, expected): + def test_next_page_token(self, config, use_deprecated_api, stream_cls, cursor_response, requests_mock, expected) -> None: test_config = use_orders_deprecated_api_config(config, use_deprecated_api) stream = stream_cls(test_config, authenticator=None) stream.page_size = 2 @@ -106,22 +117,22 @@ def test_next_page_token(self, config, use_deprecated_api, stream_cls, cursor_re "stream_cls, use_deprecated_api, next_page_token, stream_slice, expected", [ ( - Orders, - True, - None, - {"start_date":"2023-01-01 00:00:01","end_date":"2023-01-31 00:00:01"}, - {'limit': 250, 'updated_at_min': '2023-01-01 00:00:01', 'updated_at_max': '2023-01-31 00:00:01'}, + Orders, + True, + None, + {"start_date": "2023-01-01 00:00:01", "end_date": "2023-01-31 00:00:01"}, + {"limit": 250, "updated_at_min": "2023-01-01 00:00:01", "updated_at_max": "2023-01-31 00:00:01"}, ), ( - Orders, - False, - None, - {"start_date":"2023-01-01 00:00:01","end_date":"2023-01-31 00:00:01"}, - {'limit': 250, 'updated_at_min': '2023-01-01 00:00:01', 'updated_at_max': '2023-01-31 00:00:01'}, + Orders, + False, + None, + {"start_date": "2023-01-01 00:00:01", "end_date": "2023-01-31 00:00:01"}, + {"limit": 250, "updated_at_min": "2023-01-01 00:00:01", "updated_at_max": "2023-01-31 00:00:01"}, ), ], ) - def test_request_params(self, config, stream_cls, use_deprecated_api, next_page_token, stream_slice, expected): + def test_request_params(self, config, stream_cls, use_deprecated_api, next_page_token, stream_slice, expected) -> None: test_config = use_orders_deprecated_api_config(config, use_deprecated_api) stream = stream_cls(test_config, authenticator=None) result = stream.request_params(stream_slice, next_page_token) @@ -134,7 +145,7 @@ def test_request_params(self, config, stream_cls, use_deprecated_api, next_page_ (Orders, False, [{"test": 123}], [{"test": 123}]), ], ) - def test_parse_response(self, config, stream_cls, use_deprecated_api, data, requests_mock, expected): + def test_parse_response(self, config, stream_cls, use_deprecated_api, data, requests_mock, expected) -> None: test_config = use_orders_deprecated_api_config(config, use_deprecated_api) stream = stream_cls(test_config, authenticator=None) url = f"{stream.url_base}{stream.path()}" @@ -150,7 +161,7 @@ def test_parse_response(self, config, stream_cls, use_deprecated_api, data, requ (Orders, False, [{"test": 123}], [{"test": 123}]), ], ) - def get_stream_data(self, config, stream_cls, use_deprecated_api, data, requests_mock, expected): + def get_stream_data(self, config, stream_cls, use_deprecated_api, data, requests_mock, expected) -> None: test_config = use_orders_deprecated_api_config(config, use_deprecated_api) stream = stream_cls(test_config, authenticator=None) url = f"{stream.url_base}{stream.path()}" @@ -161,7 +172,7 @@ def get_stream_data(self, config, stream_cls, use_deprecated_api, data, requests class TestIncrementalStreams: - def generate_records(self, stream_name, count): + def generate_records(self, stream_name, count) -> Mapping[str, List[Mapping[str, Any]]]: result = [] for i in range(0, count): result.append({f"record_{i}": f"test_{i}"}) @@ -174,7 +185,7 @@ def generate_records(self, stream_name, count): (Orders, False, "updated_at"), ], ) - def test_cursor_field(self, config, stream_cls, use_deprecated_api, expected): + def test_cursor_field(self, config, stream_cls, use_deprecated_api, expected) -> None: test_config = use_orders_deprecated_api_config(config, use_deprecated_api) stream = stream_cls(test_config, authenticator=None) result = stream.cursor_field @@ -187,7 +198,7 @@ def test_cursor_field(self, config, stream_cls, use_deprecated_api, expected): (Orders, False, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), ], ) - def test_next_page_token(self, config, stream_cls, use_deprecated_api, cursor_response, requests_mock, expected): + def test_next_page_token(self, config, stream_cls, use_deprecated_api, cursor_response, requests_mock, expected) -> None: test_config = use_orders_deprecated_api_config(config, use_deprecated_api) stream = stream_cls(test_config, authenticator=None) stream.page_size = 2 @@ -205,24 +216,23 @@ def test_next_page_token(self, config, stream_cls, use_deprecated_api, cursor_re True, None, {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, - {'limit': 250, 'updated_at_min': '2020-01-01T00:00:00Z', 'updated_at_max': '2020-02-01T00:00:00Z'}, + {"limit": 250, "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, ), ( Orders, False, None, {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, - {'limit': 250, 'updated_at_min': '2020-01-01T00:00:00Z', 'updated_at_max': '2020-02-01T00:00:00Z'}, + {"limit": 250, "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, ), ], ) - def test_request_params(self, config, stream_cls, use_deprecated_api, next_page_token, stream_slice, expected): + def test_request_params(self, config, stream_cls, use_deprecated_api, next_page_token, stream_slice, expected) -> None: test_config = use_orders_deprecated_api_config(config, use_deprecated_api) stream = stream_cls(test_config, authenticator=None) result = stream.request_params(stream_slice, next_page_token) assert result == expected - @pytest.mark.parametrize( "stream_cls, use_deprecated_api, current_state, latest_record, expected", [ @@ -230,19 +240,19 @@ def test_request_params(self, config, stream_cls, use_deprecated_api, next_page_ (Orders, False, {"updated_at": 5}, {"updated_at": 5}, {"updated_at": 5}), ], ) - def test_get_updated_state(self, config, stream_cls,use_deprecated_api, current_state, latest_record, expected): + def test_get_updated_state(self, config, stream_cls, use_deprecated_api, current_state, latest_record, expected) -> None: test_config = use_orders_deprecated_api_config(config, use_deprecated_api) stream = stream_cls(test_config, authenticator=None) result = stream.get_updated_state(current_state, latest_record) assert result == expected - + @pytest.mark.parametrize( "stream_cls, expected", [ - (Orders, {'start_date': '2021-08-15 00:00:01', 'end_date': '2021-09-14 00:00:01'}), + (Orders, {"start_date": "2021-08-15 00:00:01", "end_date": "2021-09-14 00:00:01"}), ], ) - def test_stream_slices(self, config, stream_cls, expected): + def test_stream_slices(self, config, stream_cls, expected) -> None: stream = stream_cls(config, authenticator=None) result = list(stream.stream_slices(sync_mode=None, cursor_field=stream.cursor_field, stream_state=None)) - assert result[0] == expected \ No newline at end of file + assert result[0] == expected From 0498bbbe01c6d02cda9b49699a0cc0b1b0d56be3 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 28 Feb 2024 18:01:27 +0200 Subject: [PATCH 11/33] added integration tests for CAT missing streams --- .../acceptance-test-config.yml | 12 +-- .../source_recharge/manifest.yaml | 9 +- .../source_recharge/schemas/collections.json | 12 +++ .../unit_tests/integration/__init__.py | 0 .../unit_tests/integration/config.py | 34 ++++++ .../unit_tests/integration/pagination.py | 21 ++++ .../unit_tests/integration/request_builder.py | 49 +++++++++ .../integration/response_builder.py | 63 +++++++++++ .../integration/test_streams/__init__.py | 0 .../test_streams/test_collections.py | 57 ++++++++++ .../test_streams/test_discounts.py | 100 ++++++++++++++++++ .../integration/test_streams/test_onetimes.py | 97 +++++++++++++++++ .../unit_tests/integration/utils.py | 67 ++++++++++++ .../resource/http/response/collections.json | 26 +++++ .../resource/http/response/discounts.json | 46 ++++++++ .../resource/http/response/onetimes.json | 33 ++++++ 16 files changed, 615 insertions(+), 11 deletions(-) create mode 100644 airbyte-integrations/connectors/source-recharge/unit_tests/integration/__init__.py create mode 100644 airbyte-integrations/connectors/source-recharge/unit_tests/integration/config.py create mode 100644 airbyte-integrations/connectors/source-recharge/unit_tests/integration/pagination.py create mode 100644 airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py create mode 100644 airbyte-integrations/connectors/source-recharge/unit_tests/integration/response_builder.py create mode 100644 airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/__init__.py create mode 100644 airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/test_collections.py create mode 100644 airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/test_discounts.py create mode 100644 airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/test_onetimes.py create mode 100644 airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py create mode 100644 airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/collections.json create mode 100644 airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/discounts.json create mode 100644 airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/onetimes.json diff --git a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml index 0ecc8110e3b6..956fbe93b32a 100644 --- a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml @@ -4,11 +4,11 @@ acceptance_tests: - config_path: secrets/config.json empty_streams: - name: collections - bypass_reason: "volatile data" + bypass_reason: "The stream is tested with `Integration Tests`, since no data is available" - name: discounts - bypass_reason: "volatile data" + bypass_reason: "The stream is tested with `Integration Tests`, since no data is available" - name: onetimes - bypass_reason: "no data from stream" + bypass_reason: "The stream is tested with `Integration Tests`, since no data is available" ignored_fields: shop: - name: shop/updated_at @@ -20,11 +20,11 @@ acceptance_tests: - config_path: secrets/config_order_modern_api.json empty_streams: - name: collections - bypass_reason: "volatile data" + bypass_reason: "The stream is tested with `Integration Tests`, since no data is available" - name: discounts - bypass_reason: "volatile data" + bypass_reason: "The stream is tested with `Integration Tests`, since no data is available" - name: onetimes - bypass_reason: "no data from stream" + bypass_reason: "The stream is tested with `Integration Tests`, since no data is available" ignored_fields: shop: - name: shop/updated_at diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml b/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml index d777e3c3ade8..e38cb5272d4e 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml @@ -69,9 +69,12 @@ definitions: description: >- We should retry the SUCCESSFULL request (200), with incomplete response.content Case when: `data declared to receive` bigger than `actually received` + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: "{{ 1 if config.get('is_testing') else 10 }}" response_filters: - action: RETRY - predicate: "{{ response.content|length > 0 and headers.get('Content-Length', 0)|int > response.content|length }}" + predicate: "{{ headers.get('Content-Length', '0')|int > (response.content|length > 0)}}" requester_deprecated_api: $ref: "#/definitions/requester_base" # for deprecated retriever we should use `2021-01` api version @@ -123,10 +126,6 @@ definitions: inject_into: request_parameter type: RequestOption field_name: owner_resource - # requester: - # $ref: "#/definitions/requester_modern_api" - # request_parameters: - # owner_resource: "{{ stream_partition['owner_resource'] }}" # BASE STREAMS # FULL-REFRESH diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/collections.json b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/collections.json index 0c6b5ce3dd73..a50682e7f388 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/collections.json +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/collections.json @@ -8,6 +8,15 @@ "name": { "type": ["null", "string"] }, + "description": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, "created_at": { "type": ["null", "string"], "format": "date-time" @@ -15,6 +24,9 @@ "updated_at": { "type": ["null", "string"], "format": "date-time" + }, + "sort_order": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/__init__.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/config.py new file mode 100644 index 000000000000..2c48f96b22f4 --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/config.py @@ -0,0 +1,34 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +from __future__ import annotations + +import datetime as dt +from typing import Any, MutableMapping + +import pendulum + +START_DATE = "2023-01-01T00:00:00Z" +ACCESS_TOKEN = "test_access_token" +DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S%z" +NOW = pendulum.now(tz="utc") + + +class ConfigBuilder: + def __init__(self) -> None: + self._config: MutableMapping[str, Any] = { + "access_token": ACCESS_TOKEN, + "start_date": START_DATE, + # `is_testing` is needed to minimize the backoff time + # see `manifest.yaml > definitions > requester_base > error_handler` + "is_testing": True, + } + + def with_start_date(self, start_date: str) -> ConfigBuilder: + self._config["start_date"] = dt.datetime.strptime(start_date, DATE_TIME_FORMAT).strftime(DATE_TIME_FORMAT) + return self + + def build(self) -> MutableMapping[str, Any]: + return self._config diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/pagination.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/pagination.py new file mode 100644 index 000000000000..d996b1502b54 --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/pagination.py @@ -0,0 +1,21 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +from typing import Any, Dict + +from airbyte_cdk.test.mock_http.request import HttpRequest +from airbyte_cdk.test.mock_http.response_builder import PaginationStrategy + + +NEXT_PAGE_TOKEN = "New_Next_Page_Token" + + +class RechargePaginationStrategy(PaginationStrategy): + def __init__(self, request: HttpRequest, next_page_token: str) -> None: + self._next_page_token = next_page_token + + def update(self, response: Dict[str, Any]) -> None: + response["next_cursor"] = self._next_page_token + diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py new file mode 100644 index 000000000000..6df7245c6f8c --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py @@ -0,0 +1,49 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +from __future__ import annotations + +from typing import Any, MutableMapping + +from airbyte_cdk.test.mock_http.request import HttpRequest +import datetime as dt +from .config import ACCESS_TOKEN, DATE_TIME_FORMAT + + +def get_stream_request(stream_name: str) -> RequestBuilder: + return RequestBuilder.get_endpoint(stream_name).with_limit(250) + +class RequestBuilder: + + @classmethod + def get_endpoint(cls, endpoint: str) -> RequestBuilder: + return cls(endpoint=endpoint) + + def __init__(self, endpoint: str) -> None: + self._endpoint: str = endpoint + self._api_version: str = "2021-11" + self._query_params: MutableMapping[str, Any] = {} + + def with_limit(self, limit: int) -> RequestBuilder: + self._query_params["limit"] = limit + return self + + def with_updated_at_min(self, value: str) -> RequestBuilder: + self._query_params["updated_at_min"] = dt.datetime.strptime(value, DATE_TIME_FORMAT).strftime(DATE_TIME_FORMAT) + return self + + def with_next_page_token(self, next_page_token: str) -> RequestBuilder: + self._query_params["cursor"] = next_page_token + return self + + def build(self) -> HttpRequest: + return HttpRequest( + url=f"https://api.rechargeapps.com/{self._endpoint}", + query_params=self._query_params, + headers={ + "X-Recharge-Version": self._api_version, + "X-Recharge-Access-Token": ACCESS_TOKEN, + } + ) diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/response_builder.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/response_builder.py new file mode 100644 index 000000000000..83b916b218f5 --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/response_builder.py @@ -0,0 +1,63 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +import json +from http import HTTPStatus +from typing import Any, List, Mapping, Optional, Union + +from airbyte_cdk.test.mock_http import HttpResponse + +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) + +from .pagination import NEXT_PAGE_TOKEN, RechargePaginationStrategy +from .request_builder import get_stream_request + +def build_response( + body: Union[Mapping[str, Any], List[Mapping[str, Any]]], + status_code: HTTPStatus, + headers: Optional[Mapping[str, str]] = None, +) -> HttpResponse: + headers = headers or {} + return HttpResponse(body=json.dumps(body), status_code=status_code.value, headers=headers) + +def get_stream_response(stream_name: str) -> HttpResponseBuilder: + return create_response_builder( + response_template=find_template(stream_name, __file__), + records_path=FieldPath(stream_name), + pagination_strategy=RechargePaginationStrategy( + request=get_stream_request(stream_name).build(), next_page_token=NEXT_PAGE_TOKEN + ), + ) + +def get_stream_record( + stream_name: str, + record_id_path: str, + cursor_field: Optional[str] = None, +) -> RecordBuilder: + return create_record_builder( + response_template=find_template(stream_name, __file__), + records_path=FieldPath(stream_name), + record_id_path=FieldPath(record_id_path), + record_cursor_path=FieldPath(cursor_field) if cursor_field else None, + ) + +def successfull_incomplete_response(stream_name: str) -> HttpResponse: + body = {stream_name: [{"a": "b"}]} + response = build_response( + body=body, + # the `Content-Length` must be bigger than the actual len(data) returned + headers={"Content-Length": '10000'}, + status_code=HTTPStatus.OK, + ) + # add `content` to `response` + response.content = f"{body}" + return response diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/__init__.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/test_collections.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/test_collections.py new file mode 100644 index 000000000000..1412f5b26a23 --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/test_collections.py @@ -0,0 +1,57 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + +from unittest import TestCase +import freezegun +from airbyte_cdk.test.mock_http import HttpMocker + +from ..config import NOW +from ..request_builder import get_stream_request +from ..response_builder import ( + NEXT_PAGE_TOKEN, + successfull_incomplete_response, + get_stream_response, + get_stream_record, +) +from ..utils import config, read_full_refresh + + +_STREAM_NAME = "collections" + + +@freezegun.freeze_time(NOW.isoformat()) +class TestFullRefresh(TestCase): + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + get_stream_request(_STREAM_NAME).build(), + get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id")).build(), + ) + output = read_full_refresh(config(), _STREAM_NAME) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_multiple_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + get_stream_request(_STREAM_NAME).with_next_page_token(NEXT_PAGE_TOKEN).build(), + get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id")).build(), + ) + http_mocker.get( + get_stream_request(_STREAM_NAME).build(), + get_stream_response(_STREAM_NAME).with_pagination().with_record(get_stream_record(_STREAM_NAME, "id")).build(), + ) + + output = read_full_refresh(config(), _STREAM_NAME) + assert len(output.records) == 2 + + @HttpMocker() + def test_retry_incomplete_response_with_success_status(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + get_stream_request(_STREAM_NAME).build(), + [ + successfull_incomplete_response(_STREAM_NAME), + get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id")).build(), + ] + ) + read_full_refresh(config(), _STREAM_NAME, expected_exception=True) diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/test_discounts.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/test_discounts.py new file mode 100644 index 000000000000..9d12b7131342 --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/test_discounts.py @@ -0,0 +1,100 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +from unittest import TestCase +import freezegun +from airbyte_cdk.test.mock_http import HttpMocker + +from ..config import NOW, START_DATE +from ..request_builder import get_stream_request +from ..response_builder import ( + NEXT_PAGE_TOKEN, + successfull_incomplete_response, + get_stream_response, + get_stream_record, +) +from ..utils import ( + config, + read_full_refresh, + read_incremental, + get_cursor_value_from_state_message, +) + + +_STREAM_NAME = "discounts" +_CURSOR_FIELD = "updated_at" + + +@freezegun.freeze_time(NOW.isoformat()) +class TestFullRefresh(TestCase): + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + get_stream_request(_STREAM_NAME).with_updated_at_min(START_DATE).build(), + get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), + ) + output = read_full_refresh(config(), _STREAM_NAME) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_multiple_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + + http_mocker.get( + get_stream_request(_STREAM_NAME).with_next_page_token(NEXT_PAGE_TOKEN).build(), + get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), + ) + http_mocker.get( + get_stream_request(_STREAM_NAME).with_updated_at_min(START_DATE).build(), + get_stream_response(_STREAM_NAME).with_pagination().with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), + ) + + output = read_full_refresh(config(), _STREAM_NAME) + assert len(output.records) == 2 + + @HttpMocker() + def test_retry_incomplete_response_with_success_status(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + get_stream_request(_STREAM_NAME).with_updated_at_min(START_DATE).build(), + [ + successfull_incomplete_response(_STREAM_NAME), + get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), + ] + ) + read_full_refresh(config(), _STREAM_NAME, expected_exception=True) + + +@freezegun.freeze_time(NOW.isoformat()) +class TestIncremental(TestCase): + @HttpMocker() + def test_state_message_produced_while_read_and_state_match_latest_record(self, http_mocker: HttpMocker) -> None: + min_cursor_value = "2024-01-01T00:00:00+00:00" + max_cursor_value = "2024-02-01T00:00:00+00:00" + + http_mocker.get( + get_stream_request(_STREAM_NAME).with_updated_at_min(START_DATE).build(), + get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(min_cursor_value)).with_record( + get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(max_cursor_value)).build(), + ) + + output = read_incremental(config(), _STREAM_NAME) + test_cursor_value = get_cursor_value_from_state_message(output, _STREAM_NAME, _CURSOR_FIELD) + assert test_cursor_value == max_cursor_value + + @HttpMocker() + def test_retry_incomplete_response_with_success_status(self, http_mocker: HttpMocker) -> None: + min_cursor_value = "2024-01-01T00:00:00+00:00" + max_cursor_value = "2024-02-01T00:00:00+00:00" + + http_mocker.get( + get_stream_request(_STREAM_NAME).with_updated_at_min(START_DATE).build(), + [ + successfull_incomplete_response(_STREAM_NAME), + get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(min_cursor_value)).with_record( + get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(max_cursor_value)).build(), + ] + ) + output = read_incremental(config(), _STREAM_NAME, expected_exception=True) + test_cursor_value = get_cursor_value_from_state_message(output, _STREAM_NAME, _CURSOR_FIELD) + assert test_cursor_value == max_cursor_value \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/test_onetimes.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/test_onetimes.py new file mode 100644 index 000000000000..a0e4a497288d --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/test_onetimes.py @@ -0,0 +1,97 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +from unittest import TestCase +import freezegun +from airbyte_cdk.test.mock_http import HttpMocker + +from ..config import NOW, START_DATE +from ..request_builder import get_stream_request +from ..response_builder import ( + NEXT_PAGE_TOKEN, + successfull_incomplete_response, + get_stream_response, + get_stream_record, +) +from ..utils import ( + config, + read_full_refresh, + read_incremental, + get_cursor_value_from_state_message, +) + +_STREAM_NAME = "onetimes" +_CURSOR_FIELD = "updated_at" + + +@freezegun.freeze_time(NOW.isoformat()) +class TestFullRefresh(TestCase): + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + get_stream_request(_STREAM_NAME).with_updated_at_min(START_DATE).build(), + get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), + ) + output = read_full_refresh(config(), _STREAM_NAME) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_multiple_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + + http_mocker.get( + get_stream_request(_STREAM_NAME).with_next_page_token(NEXT_PAGE_TOKEN).build(), + get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), + ) + http_mocker.get( + get_stream_request(_STREAM_NAME).with_updated_at_min(START_DATE).build(), + get_stream_response(_STREAM_NAME).with_pagination().with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), + ) + + output = read_full_refresh(config(), _STREAM_NAME) + assert len(output.records) == 2 + + @HttpMocker() + def test_retry_incomplete_response_with_success_status(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + get_stream_request(_STREAM_NAME).with_updated_at_min(START_DATE).build(), + [ + successfull_incomplete_response(_STREAM_NAME), + get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), + ] + ) + read_full_refresh(config(), _STREAM_NAME, expected_exception=True) + + +@freezegun.freeze_time(NOW.isoformat()) +class TestIncremental(TestCase): + @HttpMocker() + def test_state_message_produced_while_read_and_state_match_latest_record(self, http_mocker: HttpMocker) -> None: + min_cursor_value = "2024-01-01T00:00:00+00:00" + max_cursor_value = "2024-02-01T00:00:00+00:00" + + http_mocker.get( + get_stream_request(_STREAM_NAME).with_updated_at_min(START_DATE).build(), + get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(min_cursor_value)).with_record( + get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(max_cursor_value)).build(), + ) + + output = read_incremental(config(), _STREAM_NAME) + test_cursor_value = get_cursor_value_from_state_message(output, _STREAM_NAME, _CURSOR_FIELD) + assert test_cursor_value == max_cursor_value + + @HttpMocker() + def test_retry_incomplete_response_with_success_status(self, http_mocker: HttpMocker) -> None: + min_cursor_value = "2024-01-01T00:00:00+00:00" + max_cursor_value = "2024-02-01T00:00:00+00:00" + + http_mocker.get( + get_stream_request(_STREAM_NAME).with_updated_at_min(START_DATE).build(), + [ + successfull_incomplete_response(_STREAM_NAME), + get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(min_cursor_value)).with_record( + get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(max_cursor_value)).build(), + ] + ) + read_incremental(config(), _STREAM_NAME, expected_exception=True) \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py new file mode 100644 index 000000000000..378d30a7a546 --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py @@ -0,0 +1,67 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + + +from typing import List, Optional, Mapping, Any + +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_protocol.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, SyncMode +from source_recharge import SourceRecharge + +from .config import ConfigBuilder + + +def config() -> ConfigBuilder: + return ConfigBuilder() + +def catalog(stream_name: str, sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(stream_name, sync_mode).build() + +def source() -> SourceRecharge: + return SourceRecharge() + +def read_output( + config_builder: ConfigBuilder, + stream_name: str, + sync_mode: SyncMode, + state: Optional[List[AirbyteStateMessage]] = None, + expected_exception: Optional[bool] = False, +) -> EntrypointOutput: + _catalog = catalog(stream_name, sync_mode) + _config = config_builder.build() + return read(source(), _config, _catalog, state, expected_exception) + +def read_full_refresh( + config_: ConfigBuilder, + stream_name: str, + expected_exception: bool = False, +) -> EntrypointOutput: + return read_output( + config_builder=config_, + stream_name=stream_name, + sync_mode=SyncMode.full_refresh, + expected_exception=expected_exception, + ) + +def read_incremental( + config_: ConfigBuilder, + stream_name: str, + state: Optional[List[AirbyteStateMessage]] = None, + expected_exception: bool = False, +) -> EntrypointOutput: + return read_output( + config_builder=config_, + stream_name=stream_name, + sync_mode=SyncMode.incremental, + state=state, + expected_exception=expected_exception, + ) + +def get_cursor_value_from_state_message( + test_output: Mapping[str, Any], + stream_name: str, + cursor_field: Optional[str] = None, +) -> str: + return test_output.most_recent_state.get(stream_name, {}).get(cursor_field) \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/collections.json b/airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/collections.json new file mode 100644 index 000000000000..6b0c4394ce56 --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/collections.json @@ -0,0 +1,26 @@ +{ + "next_cursor": null, + "previous_cursor": null, + "collections": [ + { + "id": 134129, + "created_at": "2022-03-28T12:27:03+00:00", + "name": "test_collection_134129", + "description": "kitten accessories soft.", + "sort_order": "id-asc", + "title": "Soft Kitty", + "type": "manual", + "updated_at": "2022-03-28T12:27:03+00:00" + }, + { + "id": 134136, + "created_at": "2022-03-28T15:38:27+00:00", + "name": "test_collection_134136", + "description": "cat products august 2022", + "sort_order": "title-asc", + "title": "Cats", + "type": "manual", + "updated_at": "2022-03-28T15:38:27+00:00" + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/discounts.json b/airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/discounts.json new file mode 100644 index 000000000000..86652feee230 --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/discounts.json @@ -0,0 +1,46 @@ +{ + "next_cursor": null, + "previous_cursor": null, + "discounts": [ + { + "id": 59568555, + "applies_to": { + "ids": [], + "purchase_item_type": "ALL", + "resource": null + }, + "channel_settings": { + "api": { + "can_apply": true + }, + "checkout_page": { + "can_apply": true + }, + "customer_portal": { + "can_apply": true + }, + "merchant_portal": { + "can_apply": true + } + }, + "code": "Discount1", + "created_at": "2021-07-26T19:16:17+00:00", + "ends_at": null, + "external_discount_id": { + "ecommerce": null + }, + "external_discount_source": null, + "prerequisite_subtotal_min": null, + "starts_at": null, + "status": "enabled", + "updated_at": "2024-02-01T00:00:00+00:00", + "usage_limits": { + "first_time_customer_restriction": false, + "max_subsequent_redemptions": null, + "one_application_per_customer": false + }, + "value": "100.00", + "value_type": "percentage" + } + ] + } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/onetimes.json b/airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/onetimes.json new file mode 100644 index 000000000000..09131e751015 --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/onetimes.json @@ -0,0 +1,33 @@ +{ + "next": null, + "previous": null, + "onetimes": [ + { + "id": 16909886, + "address_id": 45154492, + "created_at": "2024-01-01T00:00:00+00:00", + "customer_id": 40565990, + "external_product_id": { + "ecommerce": "4950280863846" + }, + "external_variant_id": { + "ecommerce": "32139793137766" + }, + "is_cancelled": false, + "next_charge_scheduled_at": "2025-02-01T00:00:00+00:00", + "price": "6.00", + "product_title": "ABC Shirt", + "properties": [ + { + "name": "Color", + "value": "Blue" + } + ], + "quantity": 1, + "sku": "TOM0001", + "sku_override": false, + "updated_at": "2024-02-01T00:00:00+00:00", + "variant_title": "Blue star" + } + ] + } \ No newline at end of file From e2f0fefe1f1eea5344ee82fec4efe2cd05cf232b Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 28 Feb 2024 18:06:44 +0200 Subject: [PATCH 12/33] updated version --- airbyte-integrations/connectors/source-recharge/metadata.yaml | 4 ++-- docs/integrations/sources/recharge.md | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/metadata.yaml b/airbyte-integrations/connectors/source-recharge/metadata.yaml index 31dbd5c73fc4..b325fffa947c 100644 --- a/airbyte-integrations/connectors/source-recharge/metadata.yaml +++ b/airbyte-integrations/connectors/source-recharge/metadata.yaml @@ -5,9 +5,9 @@ data: connectorSubtype: api connectorType: source connectorBuildOptions: - baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 definitionId: 45d2e135-2ede-49e1-939f-3e3ec357a65e - dockerImageTag: 1.1.5 + dockerImageTag: 2.0.0 dockerRepository: airbyte/source-recharge githubIssueLabel: source-recharge icon: recharge.svg diff --git a/docs/integrations/sources/recharge.md b/docs/integrations/sources/recharge.md index fa784da249da..ce5471c332ae 100644 --- a/docs/integrations/sources/recharge.md +++ b/docs/integrations/sources/recharge.md @@ -76,6 +76,7 @@ The Recharge connector should gracefully handle Recharge API limitations under n | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------| +| 2.0.0 | 2024-03-01 | [35450](https://github.com/airbytehq/airbyte/pull/35450) | Migrated to low-code | | 1.1.5 | 2024-02-12 | [35182](https://github.com/airbytehq/airbyte/pull/35182) | Manage dependencies with Poetry. | | 1.1.4 | 2024-02-02 | [34772](https://github.com/airbytehq/airbyte/pull/34772) | Fix airbyte-lib distribution | | 1.1.3 | 2024-01-31 | [34707](https://github.com/airbytehq/airbyte/pull/34707) | Added the UI toggle `Use 'Orders' Deprecated API` to switch between `deprecated` and `modern` api versions for `Orders` stream | From f13df0c81f185a5c903af805f0f11a563bceb0e6 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 28 Feb 2024 18:07:31 +0200 Subject: [PATCH 13/33] renamed the test folder --- .../unit_tests/integration/{test_streams => streams}/__init__.py | 0 .../integration/{test_streams => streams}/test_collections.py | 0 .../integration/{test_streams => streams}/test_discounts.py | 0 .../integration/{test_streams => streams}/test_onetimes.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename airbyte-integrations/connectors/source-recharge/unit_tests/integration/{test_streams => streams}/__init__.py (100%) rename airbyte-integrations/connectors/source-recharge/unit_tests/integration/{test_streams => streams}/test_collections.py (100%) rename airbyte-integrations/connectors/source-recharge/unit_tests/integration/{test_streams => streams}/test_discounts.py (100%) rename airbyte-integrations/connectors/source-recharge/unit_tests/integration/{test_streams => streams}/test_onetimes.py (100%) diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/__init__.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/__init__.py rename to airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/__init__.py diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/test_collections.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py similarity index 100% rename from airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/test_collections.py rename to airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/test_discounts.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py similarity index 100% rename from airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/test_discounts.py rename to airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/test_onetimes.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py similarity index 100% rename from airbyte-integrations/connectors/source-recharge/unit_tests/integration/test_streams/test_onetimes.py rename to airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py From e1327cfab9b9beb458b15826cfbba1e19d094165 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 28 Feb 2024 20:17:43 +0200 Subject: [PATCH 14/33] formatted --- .../unit_tests/integration/pagination.py | 2 - .../unit_tests/integration/request_builder.py | 8 ++- .../integration/response_builder.py | 22 ++++--- .../integration/streams/test_collections.py | 11 +--- .../integration/streams/test_discounts.py | 58 ++++++++++++------- .../integration/streams/test_onetimes.py | 57 +++++++++++------- .../unit_tests/integration/utils.py | 24 +++++--- 7 files changed, 110 insertions(+), 72 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/pagination.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/pagination.py index d996b1502b54..4522eec9675e 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/pagination.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/pagination.py @@ -8,7 +8,6 @@ from airbyte_cdk.test.mock_http.request import HttpRequest from airbyte_cdk.test.mock_http.response_builder import PaginationStrategy - NEXT_PAGE_TOKEN = "New_Next_Page_Token" @@ -18,4 +17,3 @@ def __init__(self, request: HttpRequest, next_page_token: str) -> None: def update(self, response: Dict[str, Any]) -> None: response["next_cursor"] = self._next_page_token - diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py index 6df7245c6f8c..7ddf449fec4b 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py @@ -5,16 +5,18 @@ from __future__ import annotations +import datetime as dt from typing import Any, MutableMapping from airbyte_cdk.test.mock_http.request import HttpRequest -import datetime as dt + from .config import ACCESS_TOKEN, DATE_TIME_FORMAT def get_stream_request(stream_name: str) -> RequestBuilder: return RequestBuilder.get_endpoint(stream_name).with_limit(250) + class RequestBuilder: @classmethod @@ -29,7 +31,7 @@ def __init__(self, endpoint: str) -> None: def with_limit(self, limit: int) -> RequestBuilder: self._query_params["limit"] = limit return self - + def with_updated_at_min(self, value: str) -> RequestBuilder: self._query_params["updated_at_min"] = dt.datetime.strptime(value, DATE_TIME_FORMAT).strftime(DATE_TIME_FORMAT) return self @@ -45,5 +47,5 @@ def build(self) -> HttpRequest: headers={ "X-Recharge-Version": self._api_version, "X-Recharge-Access-Token": ACCESS_TOKEN, - } + }, ) diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/response_builder.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/response_builder.py index 83b916b218f5..84d6e61322d2 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/response_builder.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/response_builder.py @@ -8,7 +8,6 @@ from typing import Any, List, Mapping, Optional, Union from airbyte_cdk.test.mock_http import HttpResponse - from airbyte_cdk.test.mock_http.response_builder import ( FieldPath, HttpResponseBuilder, @@ -21,26 +20,34 @@ from .pagination import NEXT_PAGE_TOKEN, RechargePaginationStrategy from .request_builder import get_stream_request + def build_response( body: Union[Mapping[str, Any], List[Mapping[str, Any]]], status_code: HTTPStatus, headers: Optional[Mapping[str, str]] = None, ) -> HttpResponse: headers = headers or {} - return HttpResponse(body=json.dumps(body), status_code=status_code.value, headers=headers) + return HttpResponse( + body=json.dumps(body), + status_code=status_code.value, + headers=headers, + ) + def get_stream_response(stream_name: str) -> HttpResponseBuilder: return create_response_builder( response_template=find_template(stream_name, __file__), records_path=FieldPath(stream_name), pagination_strategy=RechargePaginationStrategy( - request=get_stream_request(stream_name).build(), next_page_token=NEXT_PAGE_TOKEN + request=get_stream_request(stream_name).build(), + next_page_token=NEXT_PAGE_TOKEN, ), ) - + + def get_stream_record( - stream_name: str, - record_id_path: str, + stream_name: str, + record_id_path: str, cursor_field: Optional[str] = None, ) -> RecordBuilder: return create_record_builder( @@ -50,12 +57,13 @@ def get_stream_record( record_cursor_path=FieldPath(cursor_field) if cursor_field else None, ) + def successfull_incomplete_response(stream_name: str) -> HttpResponse: body = {stream_name: [{"a": "b"}]} response = build_response( body=body, # the `Content-Length` must be bigger than the actual len(data) returned - headers={"Content-Length": '10000'}, + headers={"Content-Length": "10000"}, status_code=HTTPStatus.OK, ) # add `content` to `response` diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py index 1412f5b26a23..1af80315a9e8 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py @@ -3,20 +3,15 @@ # from unittest import TestCase + import freezegun from airbyte_cdk.test.mock_http import HttpMocker from ..config import NOW from ..request_builder import get_stream_request -from ..response_builder import ( - NEXT_PAGE_TOKEN, - successfull_incomplete_response, - get_stream_response, - get_stream_record, -) +from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response, successfull_incomplete_response from ..utils import config, read_full_refresh - _STREAM_NAME = "collections" @@ -52,6 +47,6 @@ def test_retry_incomplete_response_with_success_status(self, http_mocker: HttpMo [ successfull_incomplete_response(_STREAM_NAME), get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id")).build(), - ] + ], ) read_full_refresh(config(), _STREAM_NAME, expected_exception=True) diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py index 9d12b7131342..74d1aa5fa8ac 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py @@ -4,24 +4,14 @@ from unittest import TestCase + import freezegun from airbyte_cdk.test.mock_http import HttpMocker from ..config import NOW, START_DATE from ..request_builder import get_stream_request -from ..response_builder import ( - NEXT_PAGE_TOKEN, - successfull_incomplete_response, - get_stream_response, - get_stream_record, -) -from ..utils import ( - config, - read_full_refresh, - read_incremental, - get_cursor_value_from_state_message, -) - +from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response, successfull_incomplete_response +from ..utils import config, get_cursor_value_from_state_message, read_full_refresh, read_incremental _STREAM_NAME = "discounts" _CURSOR_FIELD = "updated_at" @@ -40,7 +30,7 @@ def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMoc @HttpMocker() def test_given_multiple_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: - + http_mocker.get( get_stream_request(_STREAM_NAME).with_next_page_token(NEXT_PAGE_TOKEN).build(), get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), @@ -60,7 +50,7 @@ def test_retry_incomplete_response_with_success_status(self, http_mocker: HttpMo [ successfull_incomplete_response(_STREAM_NAME), get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), - ] + ], ) read_full_refresh(config(), _STREAM_NAME, expected_exception=True) @@ -74,27 +64,51 @@ def test_state_message_produced_while_read_and_state_match_latest_record(self, h http_mocker.get( get_stream_request(_STREAM_NAME).with_updated_at_min(START_DATE).build(), - get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(min_cursor_value)).with_record( - get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(max_cursor_value)).build(), + get_stream_response(_STREAM_NAME) + .with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(min_cursor_value)) + .with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(max_cursor_value)) + .build(), ) output = read_incremental(config(), _STREAM_NAME) test_cursor_value = get_cursor_value_from_state_message(output, _STREAM_NAME, _CURSOR_FIELD) assert test_cursor_value == max_cursor_value + @HttpMocker() + def test_given_multiple_pages_when_read_then_return_records_with_state(self, http_mocker: HttpMocker) -> None: + min_cursor_value = "2024-01-01T00:00:00+00:00" + max_cursor_value = "2024-02-01T00:00:00+00:00" + http_mocker.get( + get_stream_request(_STREAM_NAME).with_next_page_token(NEXT_PAGE_TOKEN).build(), + get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), + ) + http_mocker.get( + get_stream_request(_STREAM_NAME).with_updated_at_min(START_DATE).build(), + get_stream_response(_STREAM_NAME) + .with_pagination() + .with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(min_cursor_value)) + .with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(max_cursor_value)) + .build(), + ) + + output = read_incremental(config(), _STREAM_NAME) + assert len(output.records) == 3 + @HttpMocker() def test_retry_incomplete_response_with_success_status(self, http_mocker: HttpMocker) -> None: min_cursor_value = "2024-01-01T00:00:00+00:00" max_cursor_value = "2024-02-01T00:00:00+00:00" - + http_mocker.get( get_stream_request(_STREAM_NAME).with_updated_at_min(START_DATE).build(), [ successfull_incomplete_response(_STREAM_NAME), - get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(min_cursor_value)).with_record( - get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(max_cursor_value)).build(), - ] + get_stream_response(_STREAM_NAME) + .with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(min_cursor_value)) + .with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(max_cursor_value)) + .build(), + ], ) output = read_incremental(config(), _STREAM_NAME, expected_exception=True) test_cursor_value = get_cursor_value_from_state_message(output, _STREAM_NAME, _CURSOR_FIELD) - assert test_cursor_value == max_cursor_value \ No newline at end of file + assert test_cursor_value == max_cursor_value diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py index a0e4a497288d..2b449bc14799 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py @@ -4,23 +4,14 @@ from unittest import TestCase + import freezegun from airbyte_cdk.test.mock_http import HttpMocker from ..config import NOW, START_DATE from ..request_builder import get_stream_request -from ..response_builder import ( - NEXT_PAGE_TOKEN, - successfull_incomplete_response, - get_stream_response, - get_stream_record, -) -from ..utils import ( - config, - read_full_refresh, - read_incremental, - get_cursor_value_from_state_message, -) +from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response, successfull_incomplete_response +from ..utils import config, get_cursor_value_from_state_message, read_full_refresh, read_incremental _STREAM_NAME = "onetimes" _CURSOR_FIELD = "updated_at" @@ -39,7 +30,7 @@ def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMoc @HttpMocker() def test_given_multiple_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: - + http_mocker.get( get_stream_request(_STREAM_NAME).with_next_page_token(NEXT_PAGE_TOKEN).build(), get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), @@ -59,7 +50,7 @@ def test_retry_incomplete_response_with_success_status(self, http_mocker: HttpMo [ successfull_incomplete_response(_STREAM_NAME), get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), - ] + ], ) read_full_refresh(config(), _STREAM_NAME, expected_exception=True) @@ -73,25 +64,49 @@ def test_state_message_produced_while_read_and_state_match_latest_record(self, h http_mocker.get( get_stream_request(_STREAM_NAME).with_updated_at_min(START_DATE).build(), - get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(min_cursor_value)).with_record( - get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(max_cursor_value)).build(), + get_stream_response(_STREAM_NAME) + .with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(min_cursor_value)) + .with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(max_cursor_value)) + .build(), ) output = read_incremental(config(), _STREAM_NAME) test_cursor_value = get_cursor_value_from_state_message(output, _STREAM_NAME, _CURSOR_FIELD) assert test_cursor_value == max_cursor_value + @HttpMocker() + def test_given_multiple_pages_when_read_then_return_records_with_state(self, http_mocker: HttpMocker) -> None: + min_cursor_value = "2024-01-01T00:00:00+00:00" + max_cursor_value = "2024-02-01T00:00:00+00:00" + http_mocker.get( + get_stream_request(_STREAM_NAME).with_next_page_token(NEXT_PAGE_TOKEN).build(), + get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), + ) + http_mocker.get( + get_stream_request(_STREAM_NAME).with_updated_at_min(START_DATE).build(), + get_stream_response(_STREAM_NAME) + .with_pagination() + .with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(min_cursor_value)) + .with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(max_cursor_value)) + .build(), + ) + + output = read_incremental(config(), _STREAM_NAME) + assert len(output.records) == 3 + @HttpMocker() def test_retry_incomplete_response_with_success_status(self, http_mocker: HttpMocker) -> None: min_cursor_value = "2024-01-01T00:00:00+00:00" max_cursor_value = "2024-02-01T00:00:00+00:00" - + http_mocker.get( get_stream_request(_STREAM_NAME).with_updated_at_min(START_DATE).build(), [ successfull_incomplete_response(_STREAM_NAME), - get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(min_cursor_value)).with_record( - get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(max_cursor_value)).build(), - ] + get_stream_response(_STREAM_NAME) + .with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(min_cursor_value)) + .with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(max_cursor_value)) + .build(), + ], ) - read_incremental(config(), _STREAM_NAME, expected_exception=True) \ No newline at end of file + read_incremental(config(), _STREAM_NAME, expected_exception=True) diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py index 378d30a7a546..8a10ea979b07 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py @@ -3,7 +3,7 @@ # -from typing import List, Optional, Mapping, Any +from typing import Any, List, Mapping, Optional from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read @@ -16,12 +16,15 @@ def config() -> ConfigBuilder: return ConfigBuilder() + def catalog(stream_name: str, sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: return CatalogBuilder().with_stream(stream_name, sync_mode).build() + def source() -> SourceRecharge: return SourceRecharge() + def read_output( config_builder: ConfigBuilder, stream_name: str, @@ -33,9 +36,10 @@ def read_output( _config = config_builder.build() return read(source(), _config, _catalog, state, expected_exception) + def read_full_refresh( - config_: ConfigBuilder, - stream_name: str, + config_: ConfigBuilder, + stream_name: str, expected_exception: bool = False, ) -> EntrypointOutput: return read_output( @@ -44,11 +48,12 @@ def read_full_refresh( sync_mode=SyncMode.full_refresh, expected_exception=expected_exception, ) - + + def read_incremental( - config_: ConfigBuilder, - stream_name: str, - state: Optional[List[AirbyteStateMessage]] = None, + config_: ConfigBuilder, + stream_name: str, + state: Optional[List[AirbyteStateMessage]] = None, expected_exception: bool = False, ) -> EntrypointOutput: return read_output( @@ -58,10 +63,11 @@ def read_incremental( state=state, expected_exception=expected_exception, ) - + + def get_cursor_value_from_state_message( test_output: Mapping[str, Any], stream_name: str, cursor_field: Optional[str] = None, ) -> str: - return test_output.most_recent_state.get(stream_name, {}).get(cursor_field) \ No newline at end of file + return test_output.most_recent_state.get(stream_name, {}).get(cursor_field) From 3672be3664210f523158c1c14321d4ccc1978ab1 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 28 Feb 2024 22:57:07 +0200 Subject: [PATCH 15/33] updated tests --- .../source_recharge/manifest.yaml | 9 +----- .../unit_tests/integration/config.py | 3 -- .../integration/response_builder.py | 13 -------- .../integration/streams/test_collections.py | 13 +------- .../integration/streams/test_discounts.py | 32 +------------------ .../integration/streams/test_onetimes.py | 30 +---------------- 6 files changed, 4 insertions(+), 96 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml b/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml index e38cb5272d4e..8854b87d3aec 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml @@ -67,14 +67,7 @@ definitions: error_handler: type: DefaultErrorHandler description: >- - We should retry the SUCCESSFULL request (200), with incomplete response.content - Case when: `data declared to receive` bigger than `actually received` - backoff_strategies: - - type: ConstantBackoffStrategy - backoff_time_in_seconds: "{{ 1 if config.get('is_testing') else 10 }}" - response_filters: - - action: RETRY - predicate: "{{ headers.get('Content-Length', '0')|int > (response.content|length > 0)}}" + The default error handler requester_deprecated_api: $ref: "#/definitions/requester_base" # for deprecated retriever we should use `2021-01` api version diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/config.py index 2c48f96b22f4..6776c88e59f1 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/config.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/config.py @@ -21,9 +21,6 @@ def __init__(self) -> None: self._config: MutableMapping[str, Any] = { "access_token": ACCESS_TOKEN, "start_date": START_DATE, - # `is_testing` is needed to minimize the backoff time - # see `manifest.yaml > definitions > requester_base > error_handler` - "is_testing": True, } def with_start_date(self, start_date: str) -> ConfigBuilder: diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/response_builder.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/response_builder.py index 84d6e61322d2..bd2872d3db67 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/response_builder.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/response_builder.py @@ -56,16 +56,3 @@ def get_stream_record( record_id_path=FieldPath(record_id_path), record_cursor_path=FieldPath(cursor_field) if cursor_field else None, ) - - -def successfull_incomplete_response(stream_name: str) -> HttpResponse: - body = {stream_name: [{"a": "b"}]} - response = build_response( - body=body, - # the `Content-Length` must be bigger than the actual len(data) returned - headers={"Content-Length": "10000"}, - status_code=HTTPStatus.OK, - ) - # add `content` to `response` - response.content = f"{body}" - return response diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py index 1af80315a9e8..8d33879f3900 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_collections.py @@ -9,7 +9,7 @@ from ..config import NOW from ..request_builder import get_stream_request -from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response, successfull_incomplete_response +from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response from ..utils import config, read_full_refresh _STREAM_NAME = "collections" @@ -39,14 +39,3 @@ def test_given_multiple_pages_when_read_then_return_records(self, http_mocker: H output = read_full_refresh(config(), _STREAM_NAME) assert len(output.records) == 2 - - @HttpMocker() - def test_retry_incomplete_response_with_success_status(self, http_mocker: HttpMocker) -> None: - http_mocker.get( - get_stream_request(_STREAM_NAME).build(), - [ - successfull_incomplete_response(_STREAM_NAME), - get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id")).build(), - ], - ) - read_full_refresh(config(), _STREAM_NAME, expected_exception=True) diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py index 74d1aa5fa8ac..8a3990171ea0 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py @@ -10,7 +10,7 @@ from ..config import NOW, START_DATE from ..request_builder import get_stream_request -from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response, successfull_incomplete_response +from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response from ..utils import config, get_cursor_value_from_state_message, read_full_refresh, read_incremental _STREAM_NAME = "discounts" @@ -43,17 +43,6 @@ def test_given_multiple_pages_when_read_then_return_records(self, http_mocker: H output = read_full_refresh(config(), _STREAM_NAME) assert len(output.records) == 2 - @HttpMocker() - def test_retry_incomplete_response_with_success_status(self, http_mocker: HttpMocker) -> None: - http_mocker.get( - get_stream_request(_STREAM_NAME).with_updated_at_min(START_DATE).build(), - [ - successfull_incomplete_response(_STREAM_NAME), - get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), - ], - ) - read_full_refresh(config(), _STREAM_NAME, expected_exception=True) - @freezegun.freeze_time(NOW.isoformat()) class TestIncremental(TestCase): @@ -93,22 +82,3 @@ def test_given_multiple_pages_when_read_then_return_records_with_state(self, htt output = read_incremental(config(), _STREAM_NAME) assert len(output.records) == 3 - - @HttpMocker() - def test_retry_incomplete_response_with_success_status(self, http_mocker: HttpMocker) -> None: - min_cursor_value = "2024-01-01T00:00:00+00:00" - max_cursor_value = "2024-02-01T00:00:00+00:00" - - http_mocker.get( - get_stream_request(_STREAM_NAME).with_updated_at_min(START_DATE).build(), - [ - successfull_incomplete_response(_STREAM_NAME), - get_stream_response(_STREAM_NAME) - .with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(min_cursor_value)) - .with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(max_cursor_value)) - .build(), - ], - ) - output = read_incremental(config(), _STREAM_NAME, expected_exception=True) - test_cursor_value = get_cursor_value_from_state_message(output, _STREAM_NAME, _CURSOR_FIELD) - assert test_cursor_value == max_cursor_value diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py index 2b449bc14799..fa37d264c0ad 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py @@ -10,7 +10,7 @@ from ..config import NOW, START_DATE from ..request_builder import get_stream_request -from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response, successfull_incomplete_response +from ..response_builder import NEXT_PAGE_TOKEN, get_stream_record, get_stream_response from ..utils import config, get_cursor_value_from_state_message, read_full_refresh, read_incremental _STREAM_NAME = "onetimes" @@ -43,17 +43,6 @@ def test_given_multiple_pages_when_read_then_return_records(self, http_mocker: H output = read_full_refresh(config(), _STREAM_NAME) assert len(output.records) == 2 - @HttpMocker() - def test_retry_incomplete_response_with_success_status(self, http_mocker: HttpMocker) -> None: - http_mocker.get( - get_stream_request(_STREAM_NAME).with_updated_at_min(START_DATE).build(), - [ - successfull_incomplete_response(_STREAM_NAME), - get_stream_response(_STREAM_NAME).with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD)).build(), - ], - ) - read_full_refresh(config(), _STREAM_NAME, expected_exception=True) - @freezegun.freeze_time(NOW.isoformat()) class TestIncremental(TestCase): @@ -93,20 +82,3 @@ def test_given_multiple_pages_when_read_then_return_records_with_state(self, htt output = read_incremental(config(), _STREAM_NAME) assert len(output.records) == 3 - - @HttpMocker() - def test_retry_incomplete_response_with_success_status(self, http_mocker: HttpMocker) -> None: - min_cursor_value = "2024-01-01T00:00:00+00:00" - max_cursor_value = "2024-02-01T00:00:00+00:00" - - http_mocker.get( - get_stream_request(_STREAM_NAME).with_updated_at_min(START_DATE).build(), - [ - successfull_incomplete_response(_STREAM_NAME), - get_stream_response(_STREAM_NAME) - .with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(min_cursor_value)) - .with_record(get_stream_record(_STREAM_NAME, "id", _CURSOR_FIELD).with_cursor(max_cursor_value)) - .build(), - ], - ) - read_incremental(config(), _STREAM_NAME, expected_exception=True) From 19dd6c778dd7e0b9230d8db7f0c501416114d13a Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 29 Feb 2024 00:10:07 +0200 Subject: [PATCH 16/33] fixed abnormal state --- .../integration_tests/abnormal_state.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-recharge/integration_tests/abnormal_state.json index 0e0a42363b5d..9def67c7e481 100644 --- a/airbyte-integrations/connectors/source-recharge/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-recharge/integration_tests/abnormal_state.json @@ -2,49 +2,49 @@ { "type": "STREAM", "stream": { - "stream_state": { "updated_at": "2050-05-18T00:00:00" }, + "stream_state": { "updated_at": "2050-05-18T00:00:00Z" }, "stream_descriptor": { "name": "addresses" } } }, { "type": "STREAM", "stream": { - "stream_state": { "updated_at": "2050-05-18T00:00:00" }, + "stream_state": { "updated_at": "2050-05-18T00:00:00Z" }, "stream_descriptor": { "name": "charges" } } }, { "type": "STREAM", "stream": { - "stream_state": { "updated_at": "2050-05-18T00:00:00" }, + "stream_state": { "updated_at": "2050-05-18T00:00:00Z" }, "stream_descriptor": { "name": "customers" } } }, { "type": "STREAM", "stream": { - "stream_state": { "updated_at": "2050-05-18T00:00:00" }, + "stream_state": { "updated_at": "2050-05-18T00:00:00Z" }, "stream_descriptor": { "name": "discounts" } } }, { "type": "STREAM", "stream": { - "stream_state": { "updated_at": "2050-05-18T00:00:00" }, + "stream_state": { "updated_at": "2050-05-18T00:00:00Z" }, "stream_descriptor": { "name": "onetimes" } } }, { "type": "STREAM", "stream": { - "stream_state": { "updated_at": "2050-05-18T00:00:00" }, + "stream_state": { "updated_at": "2050-05-18T00:00:00Z" }, "stream_descriptor": { "name": "orders" } } }, { "type": "STREAM", "stream": { - "stream_state": { "updated_at": "2050-05-18T00:00:00" }, + "stream_state": { "updated_at": "2050-05-18T00:00:00Z" }, "stream_descriptor": { "name": "subscriptions" } } } From b3ae16cb95cc8071b36554fe285d95545e6c4331 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 29 Feb 2024 11:54:04 +0200 Subject: [PATCH 17/33] formatted --- .../source_recharge/streams.py | 2 +- .../source-recharge/unit_tests/conftest.py | 2 + .../resource/http/response/collections.json | 2 +- .../resource/http/response/discounts.json | 84 +++++++++---------- .../resource/http/response/onetimes.json | 64 +++++++------- 5 files changed, 78 insertions(+), 76 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/streams.py b/airbyte-integrations/connectors/source-recharge/source_recharge/streams.py index 0b7d2999e4e0..b2657026226d 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/streams.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/streams.py @@ -43,7 +43,7 @@ class Orders(HttpStream, ABC): # registering the default schema transformation transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - def __init__(self, config, **kwargs) -> None: + def __init__(self, config: Mapping[str, Any], **kwargs) -> None: super().__init__(**kwargs) self._start_date = config["start_date"] self.api_version = ApiVersion.DEPRECATED if config.get("use_orders_deprecated_api") else ApiVersion.MODERN diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/conftest.py b/airbyte-integrations/connectors/source-recharge/unit_tests/conftest.py index 52626e56566c..51d407d2dd18 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/conftest.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + from typing import Any, Mapping from unittest.mock import patch diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/collections.json b/airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/collections.json index 6b0c4394ce56..61c2acfefdd9 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/collections.json +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/collections.json @@ -23,4 +23,4 @@ "updated_at": "2022-03-28T15:38:27+00:00" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/discounts.json b/airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/discounts.json index 86652feee230..469a3f67d226 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/discounts.json +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/discounts.json @@ -1,46 +1,46 @@ { - "next_cursor": null, - "previous_cursor": null, - "discounts": [ - { - "id": 59568555, - "applies_to": { - "ids": [], - "purchase_item_type": "ALL", - "resource": null + "next_cursor": null, + "previous_cursor": null, + "discounts": [ + { + "id": 59568555, + "applies_to": { + "ids": [], + "purchase_item_type": "ALL", + "resource": null + }, + "channel_settings": { + "api": { + "can_apply": true }, - "channel_settings": { - "api": { - "can_apply": true - }, - "checkout_page": { - "can_apply": true - }, - "customer_portal": { - "can_apply": true - }, - "merchant_portal": { - "can_apply": true - } + "checkout_page": { + "can_apply": true }, - "code": "Discount1", - "created_at": "2021-07-26T19:16:17+00:00", - "ends_at": null, - "external_discount_id": { - "ecommerce": null + "customer_portal": { + "can_apply": true }, - "external_discount_source": null, - "prerequisite_subtotal_min": null, - "starts_at": null, - "status": "enabled", - "updated_at": "2024-02-01T00:00:00+00:00", - "usage_limits": { - "first_time_customer_restriction": false, - "max_subsequent_redemptions": null, - "one_application_per_customer": false - }, - "value": "100.00", - "value_type": "percentage" - } - ] - } \ No newline at end of file + "merchant_portal": { + "can_apply": true + } + }, + "code": "Discount1", + "created_at": "2021-07-26T19:16:17+00:00", + "ends_at": null, + "external_discount_id": { + "ecommerce": null + }, + "external_discount_source": null, + "prerequisite_subtotal_min": null, + "starts_at": null, + "status": "enabled", + "updated_at": "2024-02-01T00:00:00+00:00", + "usage_limits": { + "first_time_customer_restriction": false, + "max_subsequent_redemptions": null, + "one_application_per_customer": false + }, + "value": "100.00", + "value_type": "percentage" + } + ] +} diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/onetimes.json b/airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/onetimes.json index 09131e751015..68932c41d884 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/onetimes.json +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/resource/http/response/onetimes.json @@ -1,33 +1,33 @@ { - "next": null, - "previous": null, - "onetimes": [ - { - "id": 16909886, - "address_id": 45154492, - "created_at": "2024-01-01T00:00:00+00:00", - "customer_id": 40565990, - "external_product_id": { - "ecommerce": "4950280863846" - }, - "external_variant_id": { - "ecommerce": "32139793137766" - }, - "is_cancelled": false, - "next_charge_scheduled_at": "2025-02-01T00:00:00+00:00", - "price": "6.00", - "product_title": "ABC Shirt", - "properties": [ - { - "name": "Color", - "value": "Blue" - } - ], - "quantity": 1, - "sku": "TOM0001", - "sku_override": false, - "updated_at": "2024-02-01T00:00:00+00:00", - "variant_title": "Blue star" - } - ] - } \ No newline at end of file + "next": null, + "previous": null, + "onetimes": [ + { + "id": 16909886, + "address_id": 45154492, + "created_at": "2024-01-01T00:00:00+00:00", + "customer_id": 40565990, + "external_product_id": { + "ecommerce": "4950280863846" + }, + "external_variant_id": { + "ecommerce": "32139793137766" + }, + "is_cancelled": false, + "next_charge_scheduled_at": "2025-02-01T00:00:00+00:00", + "price": "6.00", + "product_title": "ABC Shirt", + "properties": [ + { + "name": "Color", + "value": "Blue" + } + ], + "quantity": 1, + "sku": "TOM0001", + "sku_override": false, + "updated_at": "2024-02-01T00:00:00+00:00", + "variant_title": "Blue star" + } + ] +} From a6d7c106bc51fa33c0a00d25aaac8abb07079171 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 29 Feb 2024 12:16:43 +0200 Subject: [PATCH 18/33] fixed qa checks and unit test issue --- .../connectors/source-recharge/metadata.yaml | 7 +++++++ .../connectors/source-recharge/poetry.lock | 14 +++++++------- .../connectors/source-recharge/pyproject.toml | 2 +- .../source-recharge/unit_tests/test_streams.py | 4 ++-- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/metadata.yaml b/airbyte-integrations/connectors/source-recharge/metadata.yaml index b325fffa947c..beb668c54065 100644 --- a/airbyte-integrations/connectors/source-recharge/metadata.yaml +++ b/airbyte-integrations/connectors/source-recharge/metadata.yaml @@ -23,6 +23,13 @@ data: oss: enabled: true releaseStage: generally_available + releases: + breakingChanges: + 2.0.0: + message: >- + The upgrade is considered as a Breaking Change, because it's base was moved to the Airbyte CDK (Low-code) base, + more details in this PR: https://github.com/airbytehq/airbyte/pull/35450. + upgradeDeadline: "2024-04-01" documentationUrl: https://docs.airbyte.com/integrations/sources/recharge tags: - language:python diff --git a/airbyte-integrations/connectors/source-recharge/poetry.lock b/airbyte-integrations/connectors/source-recharge/poetry.lock index b749358041e5..283a44836abf 100644 --- a/airbyte-integrations/connectors/source-recharge/poetry.lock +++ b/airbyte-integrations/connectors/source-recharge/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "airbyte-cdk" @@ -104,13 +104,13 @@ files = [ [[package]] name = "cachetools" -version = "5.3.2" +version = "5.3.3" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" files = [ - {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, - {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, + {file = "cachetools-5.3.3-py3-none-any.whl", hash = "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945"}, + {file = "cachetools-5.3.3.tar.gz", hash = "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105"}, ] [[package]] @@ -909,13 +909,13 @@ files = [ [[package]] name = "typing-extensions" -version = "4.9.0" +version = "4.10.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, - {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, + {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, + {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, ] [[package]] diff --git a/airbyte-integrations/connectors/source-recharge/pyproject.toml b/airbyte-integrations/connectors/source-recharge/pyproject.toml index c2f2326a9bb0..d5f44a5ec4e4 100644 --- a/airbyte-integrations/connectors/source-recharge/pyproject.toml +++ b/airbyte-integrations/connectors/source-recharge/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "1.1.5" +version = "2.0.0" name = "source-recharge" description = "Source implementation for Recharge." authors = [ "Airbyte ",] diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py index 8c64f31668c8..1e4ba89ffe47 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py @@ -4,7 +4,7 @@ from http import HTTPStatus -from typing import Any, List, Mapping, MutableMapping +from typing import Any, List, Mapping, MutableMapping, Union import pytest import requests @@ -88,7 +88,7 @@ def test_should_retry(self, config, http_status, headers, should_retry) -> None: class TestFullRefreshStreams: - def generate_records(self, stream_name, count) -> Mapping[str, List[Mapping[str, Any]]] | Mapping[str, Any]: + def generate_records(self, stream_name, count) -> Union[Mapping[str, List[Mapping[str, Any]]], Mapping[str, Any]]: if not stream_name: return {f"record_{1}": f"test_{1}"} result = [] From c3f492fa4009f0ba75218a2951b22738547f8ff4 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 29 Feb 2024 15:08:59 +0200 Subject: [PATCH 19/33] added empty migration guide --- docs/integrations/sources/recharge-migrations.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 docs/integrations/sources/recharge-migrations.md diff --git a/docs/integrations/sources/recharge-migrations.md b/docs/integrations/sources/recharge-migrations.md new file mode 100644 index 000000000000..a6b66ea63fe6 --- /dev/null +++ b/docs/integrations/sources/recharge-migrations.md @@ -0,0 +1,4 @@ +# Shopify Migration Guide + +## Upgrading to 2.0.0 +This version introduces no breaking changes, but the code-base changed from CDK Python to CDK Low-code. No actions are needed. \ No newline at end of file From 344f2ae597c43154a161e7d4159743841b0af013 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 29 Feb 2024 16:02:36 +0200 Subject: [PATCH 20/33] added dummy run test --- .../source-recharge/unit_tests/test_run.py | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 airbyte-integrations/connectors/source-recharge/unit_tests/test_run.py diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/test_run.py b/airbyte-integrations/connectors/source-recharge/unit_tests/test_run.py new file mode 100644 index 000000000000..f4f621f66c3a --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/test_run.py @@ -0,0 +1,68 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from argparse import Namespace +from typing import Union +from unittest.mock import MagicMock + +import pytest +from airbyte_cdk import AirbyteEntrypoint +from airbyte_cdk.models import ( + AirbyteControlConnectorConfigMessage, + AirbyteControlMessage, + AirbyteMessage, + ConnectorSpecification, + OrchestratorType, + Type, +) +from airbyte_cdk.sources import Source + + +class MockSource(Source): + def read(self, **kwargs): + pass + + def discover(self, **kwargs): + pass + + def check(self, **kwargs): + pass + + @property + def message_repository(self): + pass + + +@pytest.fixture +def entrypoint(mocker) -> AirbyteEntrypoint: + message_repository = MagicMock() + message_repository.consume_queue.side_effect = [[message for message in [MESSAGE_FROM_REPOSITORY]], [], []] + mocker.patch.object(MockSource, "message_repository", new_callable=mocker.PropertyMock, return_value=message_repository) + return AirbyteEntrypoint(MockSource()) + + +MESSAGE_FROM_REPOSITORY = AirbyteMessage( + type=Type.CONTROL, + control=AirbyteControlMessage( + type=OrchestratorType.CONNECTOR_CONFIG, + emitted_at=10, + connectorConfig=AirbyteControlConnectorConfigMessage( + config={"any config": "a config value"}, + ), + ), +) + + +def _wrap_message(submessage: Union[ConnectorSpecification]) -> str: + if isinstance(submessage, ConnectorSpecification): + message = AirbyteMessage(type=Type.SPEC, spec=submessage) + else: + raise Exception(f"Unknown message type: {submessage}") + return message.json(exclude_unset=True) + + +def test_run_spec(entrypoint: AirbyteEntrypoint, mocker): + parsed_args = Namespace(command="spec") + expected = ConnectorSpecification(connectionSpecification={"test": "test"}) + mocker.patch.object(MockSource, "spec", return_value=expected) + messages = list(entrypoint.run(parsed_args)) + assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True), _wrap_message(expected)] == messages From d5b6a9ef8c58aea6a633e460a7ec9755c6c83297 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 29 Feb 2024 16:02:59 +0200 Subject: [PATCH 21/33] removed old python base --- .../source-recharge/source_recharge/api.py | 268 ------------------ 1 file changed, 268 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-recharge/source_recharge/api.py diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py deleted file mode 100644 index aaf2cb91cf2e..000000000000 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py +++ /dev/null @@ -1,268 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from abc import ABC, abstractmethod -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union - -import pendulum -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer - - -class RechargeStream(HttpStream, ABC): - primary_key = "id" - url_base = "https://api.rechargeapps.com/" - - limit = 250 - page_num = 1 - period_in_days = 30 # Slice data request for 1 month - raise_on_http_errors = True - - # registering the default schema transformation - transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - - def __init__(self, config, **kwargs): - super().__init__(**kwargs) - self._start_date = config["start_date"] - - @property - def data_path(self): - return self.name - - @property - @abstractmethod - def api_version(self) -> str: - pass - - def request_headers( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> Mapping[str, Any]: - return {"x-recharge-version": self.api_version} - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return self.name - - @abstractmethod - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - pass - - @abstractmethod - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - pass - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - response_data = response.json() - stream_data = self.get_stream_data(response_data) - - yield from stream_data - - def get_stream_data(self, response_data: Any) -> List[dict]: - if self.data_path: - return response_data.get(self.data_path, []) - else: - return [response_data] - - def should_retry(self, response: requests.Response) -> bool: - content_length = int(response.headers.get("Content-Length", 0)) - incomplete_data_response = response.status_code == 200 and content_length > len(response.content) - - if incomplete_data_response: - return True - - return super().should_retry(response) - - def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: - start_date = (stream_state or {}).get(self.cursor_field, self._start_date) if self.cursor_field else self._start_date - - now = pendulum.now() - - # dates are inclusive, so we add 1 second so that time periods do not overlap - start_date = pendulum.parse(start_date).add(seconds=1) - - while start_date <= now: - end_date = start_date.add(days=self.period_in_days) - yield {"start_date": start_date.strftime("%Y-%m-%d %H:%M:%S"), "end_date": end_date.strftime("%Y-%m-%d %H:%M:%S")} - start_date = end_date.add(seconds=1) - - -class RechargeStreamModernAPI(RechargeStream): - api_version = "2021-11" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - cursor = response.json().get("next_cursor") - if cursor: - return {"cursor": cursor} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = {"limit": self.limit} - - # if a cursor value is passed, only limit can be passed with it! - if next_page_token: - params.update(next_page_token) - else: - params.update( - { - "updated_at_min": (stream_slice or {}).get("start_date", self._start_date), - "updated_at_max": (stream_slice or {}).get("end_date", self._start_date), - } - ) - return params - - -class RechargeStreamDeprecatedAPI(RechargeStream): - api_version = "2021-01" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - stream_data = self.get_stream_data(response.json()) - if len(stream_data) == self.limit: - self.page_num += 1 - return {"page": self.page_num} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = { - "limit": self.limit, - "updated_at_min": (stream_slice or {}).get("start_date", self._start_date), - "updated_at_max": (stream_slice or {}).get("end_date", self._start_date), - } - - if next_page_token: - params.update(next_page_token) - - return params - - -class IncrementalRechargeStream(RechargeStream, ABC): - cursor_field = "updated_at" - state_checkpoint_interval = 250 - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - latest_benchmark = latest_record[self.cursor_field] - if current_stream_state.get(self.cursor_field): - return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])} - return {self.cursor_field: latest_benchmark} - - -class Addresses(RechargeStreamModernAPI, IncrementalRechargeStream): - """ - Addresses Stream: https://developer.rechargepayments.com/v1-shopify?python#list-addresses - """ - - -class Charges(RechargeStreamModernAPI, IncrementalRechargeStream): - """ - Charges Stream: https://developer.rechargepayments.com/v1-shopify?python#list-charges - """ - - -class Collections(RechargeStreamModernAPI): - """ - Collections Stream - """ - - -class Customers(RechargeStreamModernAPI, IncrementalRechargeStream): - """ - Customers Stream: https://developer.rechargepayments.com/v1-shopify?python#list-customers - """ - - -class Discounts(RechargeStreamModernAPI, IncrementalRechargeStream): - """ - Discounts Stream: https://developer.rechargepayments.com/v1-shopify?python#list-discounts - """ - - -class Metafields(RechargeStreamModernAPI): - """ - Metafields Stream: https://developer.rechargepayments.com/v1-shopify?python#list-metafields - """ - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = {"limit": self.limit, "owner_resource": (stream_slice or {}).get("owner_resource")} - if next_page_token: - params.update(next_page_token) - - return params - - def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: - owner_resources = ["customer", "store", "subscription"] - yield from [{"owner_resource": owner} for owner in owner_resources] - - -class Onetimes(RechargeStreamModernAPI, IncrementalRechargeStream): - """ - Onetimes Stream: https://developer.rechargepayments.com/v1-shopify?python#list-onetimes - """ - - -class OrdersDeprecatedApi(RechargeStreamDeprecatedAPI, IncrementalRechargeStream): - """ - Orders Stream: https://developer.rechargepayments.com/v1-shopify?python#list-orders - Using old API version to avoid schema changes and loosing email, first_name, last_name columns, because in new version it not present - """ - - name = "orders" - - -class OrdersModernApi(RechargeStreamModernAPI, IncrementalRechargeStream): - """ - Orders Stream: https://developer.rechargepayments.com/v1-shopify?python#list-orders - Using newer API version to fetch all the data, based on the Customer's UI toggle `use_deprecated_api: FALSE`. - """ - - name = "orders" - - -class Products(RechargeStreamDeprecatedAPI): - """ - Products Stream: https://developer.rechargepayments.com/v1-shopify?python#list-products - Products endpoint has 422 error with 2021-11 API version - """ - - -class Shop(RechargeStreamDeprecatedAPI): - """ - Shop Stream: https://developer.rechargepayments.com/v1-shopify?python#shop - Shop endpoint is not available in 2021-11 API version - """ - - primary_key = ["shop", "store"] - data_path = None - - def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: - return [{}] - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - return {} - - -class Subscriptions(RechargeStreamModernAPI, IncrementalRechargeStream): - """ - Subscriptions Stream: https://developer.rechargepayments.com/v1-shopify?python#list-subscriptions - """ - - # reduce the slice date range to avoid 504 - Gateway Timeout on the Server side, - # since this stream could contain lots of data, causing the server to timeout. - # related issue: https://github.com/airbytehq/oncall/issues/3424 - period_in_days = 14 From 57010bd20decb9bc319d953a449bc8fe1d0a8e43 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 29 Feb 2024 19:25:38 +0200 Subject: [PATCH 22/33] added header to the migration guide --- docs/integrations/sources/recharge-migrations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/recharge-migrations.md b/docs/integrations/sources/recharge-migrations.md index a6b66ea63fe6..9ffe77e715e1 100644 --- a/docs/integrations/sources/recharge-migrations.md +++ b/docs/integrations/sources/recharge-migrations.md @@ -1,4 +1,4 @@ -# Shopify Migration Guide +# Recharge Migration Guide ## Upgrading to 2.0.0 This version introduces no breaking changes, but the code-base changed from CDK Python to CDK Low-code. No actions are needed. \ No newline at end of file From ac4da3def39c4c07195029fe398d50842a431188 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 29 Feb 2024 19:28:21 +0200 Subject: [PATCH 23/33] removed junk --- .../source-recharge/unit_tests/test_run.py | 68 ------------------- 1 file changed, 68 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-recharge/unit_tests/test_run.py diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/test_run.py b/airbyte-integrations/connectors/source-recharge/unit_tests/test_run.py deleted file mode 100644 index f4f621f66c3a..000000000000 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/test_run.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - -from argparse import Namespace -from typing import Union -from unittest.mock import MagicMock - -import pytest -from airbyte_cdk import AirbyteEntrypoint -from airbyte_cdk.models import ( - AirbyteControlConnectorConfigMessage, - AirbyteControlMessage, - AirbyteMessage, - ConnectorSpecification, - OrchestratorType, - Type, -) -from airbyte_cdk.sources import Source - - -class MockSource(Source): - def read(self, **kwargs): - pass - - def discover(self, **kwargs): - pass - - def check(self, **kwargs): - pass - - @property - def message_repository(self): - pass - - -@pytest.fixture -def entrypoint(mocker) -> AirbyteEntrypoint: - message_repository = MagicMock() - message_repository.consume_queue.side_effect = [[message for message in [MESSAGE_FROM_REPOSITORY]], [], []] - mocker.patch.object(MockSource, "message_repository", new_callable=mocker.PropertyMock, return_value=message_repository) - return AirbyteEntrypoint(MockSource()) - - -MESSAGE_FROM_REPOSITORY = AirbyteMessage( - type=Type.CONTROL, - control=AirbyteControlMessage( - type=OrchestratorType.CONNECTOR_CONFIG, - emitted_at=10, - connectorConfig=AirbyteControlConnectorConfigMessage( - config={"any config": "a config value"}, - ), - ), -) - - -def _wrap_message(submessage: Union[ConnectorSpecification]) -> str: - if isinstance(submessage, ConnectorSpecification): - message = AirbyteMessage(type=Type.SPEC, spec=submessage) - else: - raise Exception(f"Unknown message type: {submessage}") - return message.json(exclude_unset=True) - - -def test_run_spec(entrypoint: AirbyteEntrypoint, mocker): - parsed_args = Namespace(command="spec") - expected = ConnectorSpecification(connectionSpecification={"test": "test"}) - mocker.patch.object(MockSource, "spec", return_value=expected) - messages = list(entrypoint.run(parsed_args)) - assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True), _wrap_message(expected)] == messages From eb731eadae4895998976004a7ab8d11630d87b0a Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Mon, 11 Mar 2024 16:25:31 +0200 Subject: [PATCH 24/33] changed the cdk:python > cdk:low-code tag in metadata.yaml --- airbyte-integrations/connectors/source-recharge/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-recharge/metadata.yaml b/airbyte-integrations/connectors/source-recharge/metadata.yaml index 022c51a0d6b2..5eef01c73e8b 100644 --- a/airbyte-integrations/connectors/source-recharge/metadata.yaml +++ b/airbyte-integrations/connectors/source-recharge/metadata.yaml @@ -33,7 +33,7 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/recharge tags: - language:python - - cdk:python + - cdk:low-code ab_internal: sl: 200 ql: 400 From 972ea68da01cfae19b565c7e031ea8ca70d8e4b7 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Tue, 12 Mar 2024 14:09:28 +0200 Subject: [PATCH 25/33] added sort_by and adjusted tests accordingly --- .../connectors/source-recharge/poetry.lock | 14 ++++----- .../components/datetime_based_cursor.py | 30 +++++++++++++++++-- .../source_recharge/streams.py | 1 + .../unit_tests/integration/request_builder.py | 1 + .../unit_tests/test_streams.py | 8 ++--- 5 files changed, 41 insertions(+), 13 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/poetry.lock b/airbyte-integrations/connectors/source-recharge/poetry.lock index 283a44836abf..9b0e83c5d948 100644 --- a/airbyte-integrations/connectors/source-recharge/poetry.lock +++ b/airbyte-integrations/connectors/source-recharge/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "airbyte-cdk" @@ -481,13 +481,13 @@ files = [ [[package]] name = "packaging" -version = "23.2" +version = "24.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, - {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, + {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, + {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, ] [[package]] @@ -716,13 +716,13 @@ dev = ["pre-commit", "pytest-asyncio", "tox"] [[package]] name = "python-dateutil" -version = "2.8.2" +version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, ] [package.dependencies] diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py b/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py index 7faf13405d2a..a41b167fa73d 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py @@ -7,7 +7,7 @@ from typing import Any, Mapping, Optional from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor -from airbyte_cdk.sources.declarative.types import Record, StreamSlice +from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState @dataclass @@ -15,9 +15,11 @@ class RechargeDateTimeBasedCursor(DatetimeBasedCursor): """ Override for the default `DatetimeBasedCursor` to make self.close_slice() to produce `min` value instead of `max` value. - This is the ONLY CHANGE MADE HERE, to make the SOURCE STATE proccessed correctly: + Override for the `close_slice()` - to make the SOURCE STATE proccessed correctly: The `min` value should be determined, in the first place, since we would skip the records, if they are updated manually, by the Customer, and the range in not AFTER the STATE value, but before. + + Override for the `get_request_params()` - to guarantee the records are returned in `ASC` order. """ def __post_init__(self, parameters: Mapping[str, Any]) -> None: @@ -37,3 +39,27 @@ def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Re if cursor_value_str_by_cursor_value_datetime else None ) + + def get_request_params( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + """ + The override to add additional param to the api request to guarantee the `ASC` records order. + + Background: + There is no possability to pass multiple request params from the YAML for the incremental streams, + in addition to the `start_time_option` or similar, having them ignored those additional params, + when we have `next_page_token`, which must be the single param to be passed to satisfy the API requirements. + """ + + params = super().get_request_params( + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + ) + params["sort_by"] = "updated_at-asc" + return params diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/streams.py b/airbyte-integrations/connectors/source-recharge/source_recharge/streams.py index b2657026226d..dfdae52526eb 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/streams.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/streams.py @@ -78,6 +78,7 @@ def _update_params_with_min_max_date_range( ) -> MutableMapping[str, Any]: params.update( { + "sort_by": "updated_at-asc", "updated_at_min": (stream_slice or {}).get("start_date"), "updated_at_max": (stream_slice or {}).get("end_date"), } diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py index 7ddf449fec4b..e54bed651559 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/request_builder.py @@ -34,6 +34,7 @@ def with_limit(self, limit: int) -> RequestBuilder: def with_updated_at_min(self, value: str) -> RequestBuilder: self._query_params["updated_at_min"] = dt.datetime.strptime(value, DATE_TIME_FORMAT).strftime(DATE_TIME_FORMAT) + self._query_params["sort_by"] = "updated_at-asc" return self def with_next_page_token(self, next_page_token: str) -> RequestBuilder: diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py index 1e4ba89ffe47..636f109ff66d 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/test_streams.py @@ -121,14 +121,14 @@ def test_next_page_token(self, config, use_deprecated_api, stream_cls, cursor_re True, None, {"start_date": "2023-01-01 00:00:01", "end_date": "2023-01-31 00:00:01"}, - {"limit": 250, "updated_at_min": "2023-01-01 00:00:01", "updated_at_max": "2023-01-31 00:00:01"}, + {"limit": 250, "sort_by": "updated_at-asc", "updated_at_min": "2023-01-01 00:00:01", "updated_at_max": "2023-01-31 00:00:01"}, ), ( Orders, False, None, {"start_date": "2023-01-01 00:00:01", "end_date": "2023-01-31 00:00:01"}, - {"limit": 250, "updated_at_min": "2023-01-01 00:00:01", "updated_at_max": "2023-01-31 00:00:01"}, + {"limit": 250, "sort_by": "updated_at-asc", "updated_at_min": "2023-01-01 00:00:01", "updated_at_max": "2023-01-31 00:00:01"}, ), ], ) @@ -216,14 +216,14 @@ def test_next_page_token(self, config, stream_cls, use_deprecated_api, cursor_re True, None, {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, - {"limit": 250, "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, + {"limit": 250, "sort_by": "updated_at-asc", "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, ), ( Orders, False, None, {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, - {"limit": 250, "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, + {"limit": 250, "sort_by": "updated_at-asc", "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, ), ], ) From eae18a8bbb43776f624eceb11b8019878609734d Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Tue, 12 Mar 2024 14:11:50 +0200 Subject: [PATCH 26/33] resolved conflicts --- docs/integrations/sources/recharge.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/recharge.md b/docs/integrations/sources/recharge.md index c21e26291167..6460cedec366 100644 --- a/docs/integrations/sources/recharge.md +++ b/docs/integrations/sources/recharge.md @@ -76,7 +76,7 @@ The Recharge connector should gracefully handle Recharge API limitations under n | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------| -| 2.0.0 | 2024-03-01 | [35450](https://github.com/airbytehq/airbyte/pull/35450) | Migrated to low-code | +| 2.0.0 | 2024-03-13 | [35450](https://github.com/airbytehq/airbyte/pull/35450) | Migrated to low-code | | 1.1.6 | 2024-03-12 | [35982](https://github.com/airbytehq/airbyte/pull/35982) | Added additional `query param` to guarantee the records are in `asc` order | | 1.1.5 | 2024-02-12 | [35182](https://github.com/airbytehq/airbyte/pull/35182) | Manage dependencies with Poetry. | | 1.1.4 | 2024-02-02 | [34772](https://github.com/airbytehq/airbyte/pull/34772) | Fix airbyte-lib distribution | From 0af9d2290dad52aeabf1e5e2ccc5f131a9a9b43f Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 14 Mar 2024 18:31:19 +0200 Subject: [PATCH 27/33] updated after review --- .../connectors/source-recharge/poetry.lock | 20 ++-- .../connectors/source-recharge/pyproject.toml | 2 +- .../components/datetime_based_cursor.py | 91 +++++++++++++++---- .../source_recharge/manifest.yaml | 6 +- .../integration/streams/test_discounts.py | 2 +- .../integration/streams/test_onetimes.py | 2 +- .../unit_tests/integration/utils.py | 4 +- 7 files changed, 93 insertions(+), 34 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/poetry.lock b/airbyte-integrations/connectors/source-recharge/poetry.lock index 9b0e83c5d948..32f672870b64 100644 --- a/airbyte-integrations/connectors/source-recharge/poetry.lock +++ b/airbyte-integrations/connectors/source-recharge/poetry.lock @@ -2,13 +2,13 @@ [[package]] name = "airbyte-cdk" -version = "0.65.0" +version = "0.70.1" description = "A framework for writing Airbyte Connectors." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte-cdk-0.65.0.tar.gz", hash = "sha256:068d4419227f27d9e1d9429f27c4160850a6d6208417828055b89d1f4ef3c367"}, - {file = "airbyte_cdk-0.65.0-py3-none-any.whl", hash = "sha256:8436125d39bd058ee0ffa67d47304ab5c707b3e184dd8fcd07848959aa3b0efc"}, + {file = "airbyte-cdk-0.70.1.tar.gz", hash = "sha256:fd27815350b8155fc42afd43d005a8d321c9f309c1adaedabbb0b74e9788648f"}, + {file = "airbyte_cdk-0.70.1-py3-none-any.whl", hash = "sha256:856b51c988c8e348f53df2806d8bf929919f220f5784696cf9a9578d7eb16e72"}, ] [package.dependencies] @@ -32,8 +32,8 @@ requests-cache = "*" wcmatch = "8.4" [package.extras] -dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] -file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (>=15.0.0,<15.1.0)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (>=15.0.0,<15.1.0)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] @@ -871,18 +871,18 @@ test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "tes [[package]] name = "setuptools" -version = "69.1.1" +version = "69.2.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.1.1-py3-none-any.whl", hash = "sha256:02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56"}, - {file = "setuptools-69.1.1.tar.gz", hash = "sha256:5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8"}, + {file = "setuptools-69.2.0-py3-none-any.whl", hash = "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"}, + {file = "setuptools-69.2.0.tar.gz", hash = "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -1045,4 +1045,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9,<3.12" -content-hash = "6cc8daf1f69b1beb2f355e9d58a27076059aaccdf716b1db1c6399e5bdd7c207" +content-hash = "5a2e5306a05e0e0e9d5bf451c4e6d98a71559bfdf5e55b1358b453d19cdf021c" diff --git a/airbyte-integrations/connectors/source-recharge/pyproject.toml b/airbyte-integrations/connectors/source-recharge/pyproject.toml index d5f44a5ec4e4..dc3e752235f2 100644 --- a/airbyte-integrations/connectors/source-recharge/pyproject.toml +++ b/airbyte-integrations/connectors/source-recharge/pyproject.toml @@ -17,7 +17,7 @@ include = "source_recharge" [tool.poetry.dependencies] python = "^3.9,<3.12" -airbyte-cdk = "^0.65.0" +airbyte-cdk = ">=0.65.0" freezegun = "^1.4.0" [tool.poetry.scripts] diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py b/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py index a41b167fa73d..8dab316fae12 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py @@ -4,9 +4,11 @@ from dataclasses import dataclass -from typing import Any, Mapping, Optional +from datetime import datetime +from typing import Any, List, Mapping, Optional, Union from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState @@ -15,30 +17,83 @@ class RechargeDateTimeBasedCursor(DatetimeBasedCursor): """ Override for the default `DatetimeBasedCursor` to make self.close_slice() to produce `min` value instead of `max` value. - Override for the `close_slice()` - to make the SOURCE STATE proccessed correctly: - The `min` value should be determined, in the first place, since we would skip the records, - if they are updated manually, by the Customer, and the range in not AFTER the STATE value, but before. + 1) Override for the `close_slice()` - to make the SOURCE STATE proccessed correctly: + emit_state_from: str - `most_recent_record` or `stream_slice`, + determines what value to emit when it comes to the STATE value. - Override for the `get_request_params()` - to guarantee the records are returned in `ASC` order. + `most_recent_record` == the actual value from the RECORD + `stream_slice` == the upper boundary value of the SLICER's `stream_slice_end_value`, where the sync must be STOPPED. + + 2) Override for the `get_request_params()` - to guarantee the records are returned in `ASC` order. """ + emit_state_from: Union[InterpolatedString, str] = "stream_slice" + def __post_init__(self, parameters: Mapping[str, Any]) -> None: + self._state_selector_keys: List[str] = ["most_recent_record", "stream_slice"] + self._emit_state_from = InterpolatedString.create(self.emit_state_from, parameters=parameters).eval(self.config) + if self._emit_state_from not in self._state_selector_keys: + raise ValueError( + f"The `emit_state_from` is not one of: {self._state_selector_keys}, actual value is: `{self._emit_state_from}`." + ) super().__post_init__(parameters=parameters) + def has_emit_state_from_key_in_record(self, cursor_values: Mapping[str, Any]) -> bool: + """ + Determine whether or not we have the target key in the `cursor_values` + """ + return self._emit_state_from in cursor_values.keys() + + def get_state_to_stream_slice(self, cursor_values: Mapping[str, Any]) -> Optional[datetime]: + """ + Get the `stream_slice` cursor value from `cursor_values` + """ + return cursor_values.get("stream_slice") + + def get_state_to_emit_state_from(self, cursor_values: Mapping[str, Any]) -> Optional[datetime]: + """ + Determine whether or not we have the target key in the `cursor_values` + """ + return cursor_values.get(self._emit_state_from) + + def make_cursor_values(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> Mapping[str, Optional[datetime]]: + """ + Makes the `cursor_values` using `stream_slice` and `most_recent_record`. + """ + last_record_cursor_value = most_recent_record.get(self._cursor_field.eval(self.config)) if most_recent_record else None + stream_slice_value_end = stream_slice.get(self._partition_field_end.eval(self.config)) + cursor_values: dict = { + "most_recent_record": self.parse_date(last_record_cursor_value) if last_record_cursor_value else None, + "stream_slice": self.parse_date(stream_slice_value_end) if stream_slice_value_end else None, + } + # filter out the `NONE` STATE values from the `cursor_values` + return {key: value for key, value in cursor_values.items() if value is not None} + + def process_state(self, cursor_values: Mapping[str, Any]) -> None: + """ + Allow to get the specific STATE value which is relevant for the Source Logic. + Default is: `stream_slice` as of the default Low-Code `DatetimeBasedCursor` logic. + + Possible values are: + - cursor + - most_recent_record + - stream_slice + """ + if not self.has_emit_state_from_key_in_record(cursor_values): + # fallback to the default STATE value logic, + # if the `most_recent_record` is None or missing, + # because the `stream_slice_end_value` always exists. + return self.get_state_to_stream_slice(cursor_values) + else: + # set the STATE value to the `most_recent_record` value + return self.get_state_to_emit_state_from(cursor_values) + def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None: - last_record_cursor_value = most_recent_record.get(self.cursor_field.eval(self.config)) if most_recent_record else None - stream_slice_value_end = stream_slice.get(self.partition_field_end.eval(self.config)) - cursor_value_str_by_cursor_value_datetime = dict( - map( - lambda datetime_str: (self.parse_date(datetime_str), datetime_str), - filter(lambda item: item, [self._cursor, last_record_cursor_value, stream_slice_value_end]), - ) - ) - self._cursor = ( - cursor_value_str_by_cursor_value_datetime[min(cursor_value_str_by_cursor_value_datetime.keys())] - if cursor_value_str_by_cursor_value_datetime - else None - ) + """ + Override for the default CDK `close_slice()` method, + to provide the ability to select between the available `cursor_values`. + """ + self._cursor = self.process_state(self.make_cursor_values(stream_slice, most_recent_record)) def get_request_params( self, diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml b/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml index 8854b87d3aec..91b9af95a00b 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml @@ -1,4 +1,4 @@ -version: 0.65.0 +version: 0.70.1 definitions: # COMMON PARTS @@ -143,7 +143,9 @@ definitions: cursor_field: "updated_at" cursor_datetime_formats: - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%S" datetime_format: "%Y-%m-%dT%H:%M:%S%z" + emit_state_from: "most_recent_record" start_datetime: type: MinMaxDatetime datetime: "{{ config['start_date'] }}" @@ -189,6 +191,8 @@ definitions: $ref: "#/definitions/base_deprecated_api_stream" retriever: $ref: "#/definitions/retriever_api_deprecated" + paginator: + type: NoPagination record_selector: $ref: "#/definitions/selector" extractor: diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py index 8a3990171ea0..903456d1ddb5 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_discounts.py @@ -60,7 +60,7 @@ def test_state_message_produced_while_read_and_state_match_latest_record(self, h ) output = read_incremental(config(), _STREAM_NAME) - test_cursor_value = get_cursor_value_from_state_message(output, _STREAM_NAME, _CURSOR_FIELD) + test_cursor_value = get_cursor_value_from_state_message(output, _CURSOR_FIELD) assert test_cursor_value == max_cursor_value @HttpMocker() diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py index fa37d264c0ad..6f0fbd1b6369 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/streams/test_onetimes.py @@ -60,7 +60,7 @@ def test_state_message_produced_while_read_and_state_match_latest_record(self, h ) output = read_incremental(config(), _STREAM_NAME) - test_cursor_value = get_cursor_value_from_state_message(output, _STREAM_NAME, _CURSOR_FIELD) + test_cursor_value = get_cursor_value_from_state_message(output, _CURSOR_FIELD) assert test_cursor_value == max_cursor_value @HttpMocker() diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py index 8a10ea979b07..3c4cf5430e68 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/integration/utils.py @@ -3,6 +3,7 @@ # +from lib2to3.pgen2.literals import test from typing import Any, List, Mapping, Optional from airbyte_cdk.test.catalog_builder import CatalogBuilder @@ -67,7 +68,6 @@ def read_incremental( def get_cursor_value_from_state_message( test_output: Mapping[str, Any], - stream_name: str, cursor_field: Optional[str] = None, ) -> str: - return test_output.most_recent_state.get(stream_name, {}).get(cursor_field) + return dict(test_output.most_recent_state.stream_state).get(cursor_field) From 082019cabb03e9ec057941541d91b72ac92417d5 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 14 Mar 2024 20:15:15 +0200 Subject: [PATCH 28/33] formatted --- .../source_recharge/components/datetime_based_cursor.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py b/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py index 8dab316fae12..7f3af790123a 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py @@ -18,11 +18,10 @@ class RechargeDateTimeBasedCursor(DatetimeBasedCursor): Override for the default `DatetimeBasedCursor` to make self.close_slice() to produce `min` value instead of `max` value. 1) Override for the `close_slice()` - to make the SOURCE STATE proccessed correctly: - emit_state_from: str - `most_recent_record` or `stream_slice`, - determines what value to emit when it comes to the STATE value. + emit_state_from: str - `most_recent_record` or `stream_slice`, determines what value to emit when it comes to the STATE value. - `most_recent_record` == the actual value from the RECORD - `stream_slice` == the upper boundary value of the SLICER's `stream_slice_end_value`, where the sync must be STOPPED. + @: `most_recent_record` == the actual value from the RECORD + @: `stream_slice` == the upper boundary value of the SLICER's `stream_slice_end_value`, where the sync must be STOPPED. 2) Override for the `get_request_params()` - to guarantee the records are returned in `ASC` order. """ From d34893ba62404ac044a8d13d2d8f65ba324de163 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Mon, 18 Mar 2024 16:48:38 +0200 Subject: [PATCH 29/33] updated migration guides --- .../connectors/source-recharge/metadata.yaml | 8 +++-- .../sources/recharge-migrations.md | 29 ++++++++++++++++++- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/metadata.yaml b/airbyte-integrations/connectors/source-recharge/metadata.yaml index 5eef01c73e8b..e5ce59acdcaa 100644 --- a/airbyte-integrations/connectors/source-recharge/metadata.yaml +++ b/airbyte-integrations/connectors/source-recharge/metadata.yaml @@ -27,9 +27,11 @@ data: breakingChanges: 2.0.0: message: >- - The upgrade is considered as a Breaking Change, because it's base was moved to the Airbyte CDK (Low-code) base, - more details in this PR: https://github.com/airbytehq/airbyte/pull/35450. - upgradeDeadline: "2024-04-01" + The source `Recharge` connector is being migrated from the Python CDK to our declarative low-code CDK. + Due to changes in State format, this migration constitutes a breaking change. + After updating, please reset your source before resuming syncs. + For more information, see our migration documentation for source `Recharge`. + upgradeDeadline: "2024-04-15" documentationUrl: https://docs.airbyte.com/integrations/sources/recharge tags: - language:python diff --git a/docs/integrations/sources/recharge-migrations.md b/docs/integrations/sources/recharge-migrations.md index 9ffe77e715e1..757f73166728 100644 --- a/docs/integrations/sources/recharge-migrations.md +++ b/docs/integrations/sources/recharge-migrations.md @@ -1,4 +1,31 @@ # Recharge Migration Guide ## Upgrading to 2.0.0 -This version introduces no breaking changes, but the code-base changed from CDK Python to CDK Low-code. No actions are needed. \ No newline at end of file +We're continuously striving to enhance the quality and reliability of our connectors at Airbyte. As part of our commitment to delivering exceptional service, we are transitioning source `Recharge` from the Python Connector Development Kit (CDK) to our innovative low-code framework. This is part of a strategic move to streamline many processes across connectors, bolstering maintainability and freeing us to focus more of our efforts on improving the performance and features of our evolving platform and growing catalog. However, due to differences between the Python and low-code CDKs, this migration constitutes a breaking change. + +We’ve evolved and standardized how state is managed for incremental streams that are nested within a parent stream. This change impacts how individual states are tracked and stored for each partition, using a more structured approach to ensure the most granular and flexible state management. +This change will affect the [`addresses`, `charges`, `customers`, `discounts`, `onetimes`, `subscriptions`] streams. + +## Migration Steps + +### Refresh affected schemas and reset data + +1. Select **Connections** in the main nav bar. + 1. Select the connection(s) affected by the update. +2. Select the **Replication** tab. + 1. Select **Refresh source schema**. + 2. Select **OK**. +:::note +Any detected schema changes will be listed for your review. +::: +3. Select **Save changes** at the bottom of the page. + 1. Ensure the **Reset affected streams** option is checked. +:::note +Depending on destination type you may not be prompted to reset your data. +::: +4. Select **Save connection**. +:::note +This will reset the data in your destination and initiate a fresh sync. +::: + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). From f1ee0d03e7f9263de3d27b25a7a7d84610a769f5 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Mon, 18 Mar 2024 16:56:02 +0200 Subject: [PATCH 30/33] updated migration guides including OSS instructions --- .../sources/recharge-migrations.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/integrations/sources/recharge-migrations.md b/docs/integrations/sources/recharge-migrations.md index 757f73166728..c91be1a7e23e 100644 --- a/docs/integrations/sources/recharge-migrations.md +++ b/docs/integrations/sources/recharge-migrations.md @@ -7,6 +7,31 @@ We’ve evolved and standardized how state is managed for incremental streams th This change will affect the [`addresses`, `charges`, `customers`, `discounts`, `onetimes`, `subscriptions`] streams. ## Migration Steps +### For Airbyte Open Source: Update the local connector image + +Airbyte Open Source users must manually update the connector image in their local registry before proceeding with the migration. To do so: + +1. Select **Settings** in the main navbar. + 1. Select **Sources**. +2. Find `Recharge` in the list of connectors. + +:::note +You will see two versions listed, the current in-use version and the latest version available. +::: + +3. Select **Change** to update your OSS version to the latest available version. + +### Update the connector version + +1. Select **Sources** in the main navbar. +2. Select the instance of the connector you wish to upgrade. + +:::note +Each instance of the connector must be updated separately. If you have created multiple instances of a connector, updating one will not affect the others. +::: + +3. Select **Upgrade** + 1. Follow the prompt to confirm you are ready to upgrade to the new version. ### Refresh affected schemas and reset data From dfe3eb2ee79ea0ba77a313ddf829b86ab035efa8 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 20 Mar 2024 16:57:23 +0200 Subject: [PATCH 31/33] removed CustomComponents, moved to the latest CDK, removed breaking changes mentions --- .../connectors/source-recharge/metadata.yaml | 9 -- .../connectors/source-recharge/poetry.lock | 8 +- .../connectors/source-recharge/pyproject.toml | 2 +- .../components/datetime_based_cursor.py | 119 ------------------ .../source_recharge/manifest.yaml | 6 +- .../sources/recharge-migrations.md | 56 --------- 6 files changed, 7 insertions(+), 193 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py delete mode 100644 docs/integrations/sources/recharge-migrations.md diff --git a/airbyte-integrations/connectors/source-recharge/metadata.yaml b/airbyte-integrations/connectors/source-recharge/metadata.yaml index e5ce59acdcaa..645c1d0fd552 100644 --- a/airbyte-integrations/connectors/source-recharge/metadata.yaml +++ b/airbyte-integrations/connectors/source-recharge/metadata.yaml @@ -23,15 +23,6 @@ data: oss: enabled: true releaseStage: generally_available - releases: - breakingChanges: - 2.0.0: - message: >- - The source `Recharge` connector is being migrated from the Python CDK to our declarative low-code CDK. - Due to changes in State format, this migration constitutes a breaking change. - After updating, please reset your source before resuming syncs. - For more information, see our migration documentation for source `Recharge`. - upgradeDeadline: "2024-04-15" documentationUrl: https://docs.airbyte.com/integrations/sources/recharge tags: - language:python diff --git a/airbyte-integrations/connectors/source-recharge/poetry.lock b/airbyte-integrations/connectors/source-recharge/poetry.lock index 32f672870b64..a997948dfbca 100644 --- a/airbyte-integrations/connectors/source-recharge/poetry.lock +++ b/airbyte-integrations/connectors/source-recharge/poetry.lock @@ -2,13 +2,13 @@ [[package]] name = "airbyte-cdk" -version = "0.70.1" +version = "0.72.2" description = "A framework for writing Airbyte Connectors." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte-cdk-0.70.1.tar.gz", hash = "sha256:fd27815350b8155fc42afd43d005a8d321c9f309c1adaedabbb0b74e9788648f"}, - {file = "airbyte_cdk-0.70.1-py3-none-any.whl", hash = "sha256:856b51c988c8e348f53df2806d8bf929919f220f5784696cf9a9578d7eb16e72"}, + {file = "airbyte-cdk-0.72.2.tar.gz", hash = "sha256:3c06ed9c1436967ffde77b51814772dbbd79745d610bc2fe400dff9c4d7a9877"}, + {file = "airbyte_cdk-0.72.2-py3-none-any.whl", hash = "sha256:8d50773fe9ffffe9be8d6c2d2fcb10c50153833053b3ef4283fcb39c544dc4b9"}, ] [package.dependencies] @@ -1045,4 +1045,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9,<3.12" -content-hash = "5a2e5306a05e0e0e9d5bf451c4e6d98a71559bfdf5e55b1358b453d19cdf021c" +content-hash = "6c8e1b56b7d37fab639950309e88d5e32cf513f66bceb08ab8d7950c535db192" diff --git a/airbyte-integrations/connectors/source-recharge/pyproject.toml b/airbyte-integrations/connectors/source-recharge/pyproject.toml index dc3e752235f2..cc91dd906725 100644 --- a/airbyte-integrations/connectors/source-recharge/pyproject.toml +++ b/airbyte-integrations/connectors/source-recharge/pyproject.toml @@ -17,7 +17,7 @@ include = "source_recharge" [tool.poetry.dependencies] python = "^3.9,<3.12" -airbyte-cdk = ">=0.65.0" +airbyte-cdk = "^0" freezegun = "^1.4.0" [tool.poetry.scripts] diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py b/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py deleted file mode 100644 index 7f3af790123a..000000000000 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py +++ /dev/null @@ -1,119 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from dataclasses import dataclass -from datetime import datetime -from typing import Any, List, Mapping, Optional, Union - -from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor -from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString -from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState - - -@dataclass -class RechargeDateTimeBasedCursor(DatetimeBasedCursor): - """ - Override for the default `DatetimeBasedCursor` to make self.close_slice() to produce `min` value instead of `max` value. - - 1) Override for the `close_slice()` - to make the SOURCE STATE proccessed correctly: - emit_state_from: str - `most_recent_record` or `stream_slice`, determines what value to emit when it comes to the STATE value. - - @: `most_recent_record` == the actual value from the RECORD - @: `stream_slice` == the upper boundary value of the SLICER's `stream_slice_end_value`, where the sync must be STOPPED. - - 2) Override for the `get_request_params()` - to guarantee the records are returned in `ASC` order. - """ - - emit_state_from: Union[InterpolatedString, str] = "stream_slice" - - def __post_init__(self, parameters: Mapping[str, Any]) -> None: - self._state_selector_keys: List[str] = ["most_recent_record", "stream_slice"] - self._emit_state_from = InterpolatedString.create(self.emit_state_from, parameters=parameters).eval(self.config) - if self._emit_state_from not in self._state_selector_keys: - raise ValueError( - f"The `emit_state_from` is not one of: {self._state_selector_keys}, actual value is: `{self._emit_state_from}`." - ) - super().__post_init__(parameters=parameters) - - def has_emit_state_from_key_in_record(self, cursor_values: Mapping[str, Any]) -> bool: - """ - Determine whether or not we have the target key in the `cursor_values` - """ - return self._emit_state_from in cursor_values.keys() - - def get_state_to_stream_slice(self, cursor_values: Mapping[str, Any]) -> Optional[datetime]: - """ - Get the `stream_slice` cursor value from `cursor_values` - """ - return cursor_values.get("stream_slice") - - def get_state_to_emit_state_from(self, cursor_values: Mapping[str, Any]) -> Optional[datetime]: - """ - Determine whether or not we have the target key in the `cursor_values` - """ - return cursor_values.get(self._emit_state_from) - - def make_cursor_values(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> Mapping[str, Optional[datetime]]: - """ - Makes the `cursor_values` using `stream_slice` and `most_recent_record`. - """ - last_record_cursor_value = most_recent_record.get(self._cursor_field.eval(self.config)) if most_recent_record else None - stream_slice_value_end = stream_slice.get(self._partition_field_end.eval(self.config)) - cursor_values: dict = { - "most_recent_record": self.parse_date(last_record_cursor_value) if last_record_cursor_value else None, - "stream_slice": self.parse_date(stream_slice_value_end) if stream_slice_value_end else None, - } - # filter out the `NONE` STATE values from the `cursor_values` - return {key: value for key, value in cursor_values.items() if value is not None} - - def process_state(self, cursor_values: Mapping[str, Any]) -> None: - """ - Allow to get the specific STATE value which is relevant for the Source Logic. - Default is: `stream_slice` as of the default Low-Code `DatetimeBasedCursor` logic. - - Possible values are: - - cursor - - most_recent_record - - stream_slice - """ - if not self.has_emit_state_from_key_in_record(cursor_values): - # fallback to the default STATE value logic, - # if the `most_recent_record` is None or missing, - # because the `stream_slice_end_value` always exists. - return self.get_state_to_stream_slice(cursor_values) - else: - # set the STATE value to the `most_recent_record` value - return self.get_state_to_emit_state_from(cursor_values) - - def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None: - """ - Override for the default CDK `close_slice()` method, - to provide the ability to select between the available `cursor_values`. - """ - self._cursor = self.process_state(self.make_cursor_values(stream_slice, most_recent_record)) - - def get_request_params( - self, - *, - stream_state: Optional[StreamState] = None, - stream_slice: Optional[StreamSlice] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Mapping[str, Any]: - """ - The override to add additional param to the api request to guarantee the `ASC` records order. - - Background: - There is no possability to pass multiple request params from the YAML for the incremental streams, - in addition to the `start_time_option` or similar, having them ignored those additional params, - when we have `next_page_token`, which must be the single param to be passed to satisfy the API requirements. - """ - - params = super().get_request_params( - stream_state=stream_state, - stream_slice=stream_slice, - next_page_token=next_page_token, - ) - params["sort_by"] = "updated_at-asc" - return params diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml b/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml index 91b9af95a00b..6224d9546d70 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml @@ -1,4 +1,4 @@ -version: 0.70.1 +version: 0.72.2 definitions: # COMMON PARTS @@ -138,14 +138,12 @@ definitions: base_incremental_stream: $ref: "#/definitions/base_modern_api_stream" incremental_sync: - type: CustomIncrementalSync - class_name: source_recharge.components.datetime_based_cursor.RechargeDateTimeBasedCursor + type: DatetimeBasedCursor cursor_field: "updated_at" cursor_datetime_formats: - "%Y-%m-%dT%H:%M:%S%z" - "%Y-%m-%dT%H:%M:%S" datetime_format: "%Y-%m-%dT%H:%M:%S%z" - emit_state_from: "most_recent_record" start_datetime: type: MinMaxDatetime datetime: "{{ config['start_date'] }}" diff --git a/docs/integrations/sources/recharge-migrations.md b/docs/integrations/sources/recharge-migrations.md deleted file mode 100644 index c91be1a7e23e..000000000000 --- a/docs/integrations/sources/recharge-migrations.md +++ /dev/null @@ -1,56 +0,0 @@ -# Recharge Migration Guide - -## Upgrading to 2.0.0 -We're continuously striving to enhance the quality and reliability of our connectors at Airbyte. As part of our commitment to delivering exceptional service, we are transitioning source `Recharge` from the Python Connector Development Kit (CDK) to our innovative low-code framework. This is part of a strategic move to streamline many processes across connectors, bolstering maintainability and freeing us to focus more of our efforts on improving the performance and features of our evolving platform and growing catalog. However, due to differences between the Python and low-code CDKs, this migration constitutes a breaking change. - -We’ve evolved and standardized how state is managed for incremental streams that are nested within a parent stream. This change impacts how individual states are tracked and stored for each partition, using a more structured approach to ensure the most granular and flexible state management. -This change will affect the [`addresses`, `charges`, `customers`, `discounts`, `onetimes`, `subscriptions`] streams. - -## Migration Steps -### For Airbyte Open Source: Update the local connector image - -Airbyte Open Source users must manually update the connector image in their local registry before proceeding with the migration. To do so: - -1. Select **Settings** in the main navbar. - 1. Select **Sources**. -2. Find `Recharge` in the list of connectors. - -:::note -You will see two versions listed, the current in-use version and the latest version available. -::: - -3. Select **Change** to update your OSS version to the latest available version. - -### Update the connector version - -1. Select **Sources** in the main navbar. -2. Select the instance of the connector you wish to upgrade. - -:::note -Each instance of the connector must be updated separately. If you have created multiple instances of a connector, updating one will not affect the others. -::: - -3. Select **Upgrade** - 1. Follow the prompt to confirm you are ready to upgrade to the new version. - -### Refresh affected schemas and reset data - -1. Select **Connections** in the main nav bar. - 1. Select the connection(s) affected by the update. -2. Select the **Replication** tab. - 1. Select **Refresh source schema**. - 2. Select **OK**. -:::note -Any detected schema changes will be listed for your review. -::: -3. Select **Save changes** at the bottom of the page. - 1. Ensure the **Reset affected streams** option is checked. -:::note -Depending on destination type you may not be prompted to reset your data. -::: -4. Select **Save connection**. -:::note -This will reset the data in your destination and initiate a fresh sync. -::: - -For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). From 7b42e9d86b372fe87de81bdeaade6aecdecf31c3 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 20 Mar 2024 17:22:38 +0200 Subject: [PATCH 32/33] corrected version bumps --- airbyte-integrations/connectors/source-recharge/metadata.yaml | 2 +- airbyte-integrations/connectors/source-recharge/pyproject.toml | 2 +- docs/integrations/sources/recharge.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-recharge/metadata.yaml b/airbyte-integrations/connectors/source-recharge/metadata.yaml index 645c1d0fd552..0b87105819eb 100644 --- a/airbyte-integrations/connectors/source-recharge/metadata.yaml +++ b/airbyte-integrations/connectors/source-recharge/metadata.yaml @@ -7,7 +7,7 @@ data: connectorBuildOptions: baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 definitionId: 45d2e135-2ede-49e1-939f-3e3ec357a65e - dockerImageTag: 2.0.0 + dockerImageTag: 1.2.0 dockerRepository: airbyte/source-recharge githubIssueLabel: source-recharge icon: recharge.svg diff --git a/airbyte-integrations/connectors/source-recharge/pyproject.toml b/airbyte-integrations/connectors/source-recharge/pyproject.toml index cc91dd906725..5bc47b80b6ea 100644 --- a/airbyte-integrations/connectors/source-recharge/pyproject.toml +++ b/airbyte-integrations/connectors/source-recharge/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "2.0.0" +version = "1.2.0" name = "source-recharge" description = "Source implementation for Recharge." authors = [ "Airbyte ",] diff --git a/docs/integrations/sources/recharge.md b/docs/integrations/sources/recharge.md index 6460cedec366..307c4256dbfb 100644 --- a/docs/integrations/sources/recharge.md +++ b/docs/integrations/sources/recharge.md @@ -76,7 +76,7 @@ The Recharge connector should gracefully handle Recharge API limitations under n | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------| -| 2.0.0 | 2024-03-13 | [35450](https://github.com/airbytehq/airbyte/pull/35450) | Migrated to low-code | +| 1.2.0 | 2024-03-13 | [35450](https://github.com/airbytehq/airbyte/pull/35450) | Migrated to low-code | | 1.1.6 | 2024-03-12 | [35982](https://github.com/airbytehq/airbyte/pull/35982) | Added additional `query param` to guarantee the records are in `asc` order | | 1.1.5 | 2024-02-12 | [35182](https://github.com/airbytehq/airbyte/pull/35182) | Manage dependencies with Poetry. | | 1.1.4 | 2024-02-02 | [34772](https://github.com/airbytehq/airbyte/pull/34772) | Fix airbyte-lib distribution | From a1d74e02420faebe3602229b9a9e7248ed500995 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 20 Mar 2024 20:35:06 +0200 Subject: [PATCH 33/33] fixed unit_tests --- .../components/datetime_based_cursor.py | 70 +++++++++++++++++++ .../source_recharge/manifest.yaml | 3 +- 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py b/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py new file mode 100644 index 000000000000..7957a3c0d906 --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/components/datetime_based_cursor.py @@ -0,0 +1,70 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from dataclasses import dataclass +from datetime import datetime +from typing import Any, List, Mapping, Optional, Union + +from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState + + +@dataclass +class RechargeDateTimeBasedCursor(DatetimeBasedCursor): + """ + Override for the default `DatetimeBasedCursor`. + + `get_request_params()` - to guarantee the records are returned in `ASC` order. + + Currently the `HttpRequester` couldn't handle the case when, + we need to omit all other `request_params` but `next_page_token` param, + typically when the `CursorPagination` straregy is applied. + + We should have the `request_parameters` structure like this, or similar to either keep or omit the parameter, + based on the paginated result: + ``` + HttpRequester: + ... + request_parameters: + # The `sort_by` param, will be omitted intentionaly on the paginated result + - sort_by: "updated_at-asc" + ignore_on_pagination: true + # the `some_other_param` param, will be kept on the paginated result + - some_other_param: "string_value" + ignore_on_pagination: false + ``` + + Because there is a `ignore_stream_slicer_parameters_on_paginated_requests` set to True for the `SimpleRetriever`, + we are able to omit everthing what we pass from the `DatetimeBasedCursor.get_request_params()` having the initial request as expected, + all subsequent requests are made based on Paginated Results. + """ + + def __post_init__(self, parameters: Mapping[str, Any]) -> None: + super().__post_init__(parameters=parameters) + + def get_request_params( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + """ + The override to add additional param to the api request to guarantee the `ASC` records order. + + Background: + There is no possability to pass multiple request params from the YAML for the incremental streams, + in addition to the `start_time_option` or similar, having them ignored those additional params, + when we have `next_page_token`, which must be the single param to be passed to satisfy the API requirements. + """ + + params = super().get_request_params( + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + ) + params["sort_by"] = "updated_at-asc" + return params diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml b/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml index 6224d9546d70..45f3b06f5002 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/manifest.yaml @@ -138,7 +138,8 @@ definitions: base_incremental_stream: $ref: "#/definitions/base_modern_api_stream" incremental_sync: - type: DatetimeBasedCursor + type: CustomIncrementalSync + class_name: source_recharge.components.datetime_based_cursor.RechargeDateTimeBasedCursor cursor_field: "updated_at" cursor_datetime_formats: - "%Y-%m-%dT%H:%M:%S%z"