From 07dddba5ba318a01623956d5cc469c8af398b270 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Mon, 4 Mar 2024 13:54:17 +0200 Subject: [PATCH 01/23] changed version --- .../connectors/source-zendesk-chat/README.md | 2 +- .../source-zendesk-chat/metadata.yaml | 10 ++- .../source-zendesk-chat/poetry.lock | 86 +++++++++++-------- .../source-zendesk-chat/pyproject.toml | 4 +- 4 files changed, 62 insertions(+), 40 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/README.md b/airbyte-integrations/connectors/source-zendesk-chat/README.md index f7d40d3e06a8b..411735aa8b10c 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/README.md +++ b/airbyte-integrations/connectors/source-zendesk-chat/README.md @@ -30,7 +30,7 @@ See `sample_files/sample_config.json` for a sample config file. poetry run source-zendesk-chat spec poetry run source-zendesk-chat check --config secrets/config.json poetry run source-zendesk-chat discover --config secrets/config.json -poetry run source-zendesk-chat read --config secrets/config.json --catalog sample_files/configured_catalog.json +poetry run source-zendesk-chat read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` ### Running unit tests diff --git a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml index 1aa58d5146b59..c3e9a65b45168 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml @@ -6,11 +6,11 @@ data: hosts: - zopim.com connectorBuildOptions: - baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4 - dockerImageTag: 0.2.2 + dockerImageTag: 1.0.0 dockerRepository: airbyte/source-zendesk-chat documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-chat githubIssueLabel: source-zendesk-chat @@ -27,6 +27,12 @@ data: oss: enabled: true releaseStage: generally_available + releases: + breakingChanges: + 1.0.0: + message: >- + This is the upgrade! Wow! + upgradeDeadline: "2024-04-10" supportLevel: certified tags: - language:python diff --git a/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock b/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock index 4035ad70602bc..d551758b15571 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock +++ b/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock @@ -2,17 +2,17 @@ [[package]] name = "airbyte-cdk" -version = "0.51.41" +version = "0.65.0" description = "A framework for writing Airbyte Connectors." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte-cdk-0.51.41.tar.gz", hash = "sha256:cce614d67872cf66a151e5b72d70f4bf26e2a1ce672c7abfc15a5cb4e45d8429"}, - {file = "airbyte_cdk-0.51.41-py3-none-any.whl", hash = "sha256:bbf82a45d9ec97c4a92b85e3312b327f8060fffec1f7c7ea7dfa720f9adcc13b"}, + {file = "airbyte-cdk-0.65.0.tar.gz", hash = "sha256:068d4419227f27d9e1d9429f27c4160850a6d6208417828055b89d1f4ef3c367"}, + {file = "airbyte_cdk-0.65.0-py3-none-any.whl", hash = "sha256:8436125d39bd058ee0ffa67d47304ab5c707b3e184dd8fcd07848959aa3b0efc"}, ] [package.dependencies] -airbyte-protocol-models = "0.4.2" +airbyte-protocol-models = "0.5.1" backoff = "*" cachetools = "*" Deprecated = ">=1.2,<2.0" @@ -22,8 +22,9 @@ isodate = ">=0.6.1,<0.7.0" Jinja2 = ">=3.1.2,<3.2.0" jsonref = ">=0.2,<1.0" jsonschema = ">=3.2.0,<3.3.0" -pendulum = "*" +pendulum = "<3.0.0" pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" python-dateutil = "*" PyYAML = ">=6.0.1" requests = "*" @@ -31,20 +32,20 @@ requests-cache = "*" wcmatch = "8.4" [package.extras] -dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pyarrow (==12.0.1)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)"] -file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "pyarrow (==12.0.1)"] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] [[package]] name = "airbyte-protocol-models" -version = "0.4.2" +version = "0.5.1" description = "Declares the Airbyte Protocol." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte_protocol_models-0.4.2-py3-none-any.whl", hash = "sha256:d3bbb14d4af9483bd7b08f5eb06f87e7113553bf4baed3998af95be873a0d821"}, - {file = "airbyte_protocol_models-0.4.2.tar.gz", hash = "sha256:67b149d4812f8fdb88396b161274aa73cf0e16f22e35ce44f2bfc4d47e51915c"}, + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, ] [package.dependencies] @@ -103,13 +104,13 @@ files = [ [[package]] name = "cachetools" -version = "5.3.2" +version = "5.3.3" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" files = [ - {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, - {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, + {file = "cachetools-5.3.3-py3-none-any.whl", hash = "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945"}, + {file = "cachetools-5.3.3.tar.gz", hash = "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105"}, ] [[package]] @@ -602,6 +603,21 @@ typing-extensions = ">=4.2.0" dotenv = ["python-dotenv (>=0.10.4)"] email = ["email-validator (>=1.0.3)"] +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + [[package]] name = "pyrsistent" version = "0.20.0" @@ -686,13 +702,13 @@ dev = ["pre-commit", "pytest-asyncio", "tox"] [[package]] name = "python-dateutil" -version = "2.8.2" +version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, ] [package.dependencies] @@ -792,13 +808,13 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requests-cache" -version = "1.1.1" +version = "1.2.0" description = "A persistent cache for python requests" optional = false -python-versions = ">=3.7,<4.0" +python-versions = ">=3.8" files = [ - {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, - {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, + {file = "requests_cache-1.2.0-py3-none-any.whl", hash = "sha256:490324301bf0cb924ff4e6324bd2613453e7e1f847353928b08adb0fdfb7f722"}, + {file = "requests_cache-1.2.0.tar.gz", hash = "sha256:db1c709ca343cc1cd5b6c8b1a5387298eceed02306a6040760db538c885e3838"}, ] [package.dependencies] @@ -810,15 +826,15 @@ url-normalize = ">=1.4" urllib3 = ">=1.25.5" [package.extras] -all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=6.0.1)", "redis (>=3)", "ujson (>=5.4)"] bson = ["bson (>=0.5)"] -docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.9)"] dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] json = ["ujson (>=5.4)"] mongodb = ["pymongo (>=3)"] redis = ["redis (>=3)"] security = ["itsdangerous (>=2.0)"] -yaml = ["pyyaml (>=5.4)"] +yaml = ["pyyaml (>=6.0.1)"] [[package]] name = "requests-mock" @@ -841,19 +857,19 @@ test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "tes [[package]] name = "setuptools" -version = "69.1.0" +version = "69.1.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, - {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, + {file = "setuptools-69.1.1-py3-none-any.whl", hash = "sha256:02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56"}, + {file = "setuptools-69.1.1.tar.gz", hash = "sha256:5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" @@ -879,13 +895,13 @@ files = [ [[package]] name = "typing-extensions" -version = "4.9.0" +version = "4.10.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, - {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, + {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, + {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, ] [[package]] @@ -904,13 +920,13 @@ six = "*" [[package]] name = "urllib3" -version = "2.2.0" +version = "2.2.1" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.8" files = [ - {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, - {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, ] [package.extras] @@ -1015,4 +1031,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9,<3.12" -content-hash = "e55b65b435ed00315a8288393c1fb2adde5904ae32b5aed66f133bdb721a6991" +content-hash = "bacda57c2899ee07da6cd98d63266c42914e1735baad9c74660ffdc7cb61cd02" diff --git a/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml b/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml index f47dbc02c81db..4d2622ef7422b 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml +++ b/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "0.2.2" +version = "1.0.0" name = "source-zendesk-chat" description = "Source implementation for Zendesk Chat." authors = [ "Airbyte ",] @@ -17,7 +17,7 @@ include = "source_zendesk_chat" [tool.poetry.dependencies] python = "^3.9,<3.12" -airbyte-cdk = "==0.51.41" +airbyte-cdk = "^0.65.0" pendulum = "==2.1.2" [tool.poetry.scripts] From 8c7468f3e90406839b6ceb620d93d583c5fbbad7 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Mon, 4 Mar 2024 17:34:22 +0200 Subject: [PATCH 02/23] updated CAT config --- .../connectors/source-zendesk-chat/acceptance-test-config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml index 37352d28dabec..b0acaaaefee0a 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml @@ -24,11 +24,9 @@ acceptance_tests: - config_path: "secrets/config.json" expect_records: path: "integration_tests/expected_records.txt" - fail_on_extra_columns: false - config_path: "secrets/config_oauth.json" expect_records: path: "integration_tests/expected_records.txt" - fail_on_extra_columns: false incremental: tests: - config_path: "secrets/config.json" From f938394b4d10e09f27e5cf8cc21a60a9c625110d Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Tue, 5 Mar 2024 17:37:44 +0200 Subject: [PATCH 03/23] added manifest and custom components --- .../components/id_incremental.py | 111 +++++++ .../components/id_offset_pagination.py | 38 +++ .../source_zendesk_chat/manifest.yaml | 273 ++++++++++++++++++ .../source_zendesk_chat/source.py | 62 +--- 4 files changed, 432 insertions(+), 52 deletions(-) create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental.py new file mode 100644 index 0000000000000..ead6bd205cc27 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental.py @@ -0,0 +1,111 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import InitVar, dataclass, field +from typing import Any, Iterable, Mapping, Optional, Union + +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type +from airbyte_cdk.sources.declarative.incremental.cursor import Cursor +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation +from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType +from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState +from airbyte_cdk.sources.message import MessageRepository + + +@dataclass +class IdIncrementalCursor(Cursor): + """ + Custom Incremental Cursor implementation to provide the ability to pull data using `id`(int) as cursor. + More info: https://developer.zendesk.com/api-reference/live-chat/chat-api/agents/#parameters + + Attributes: + config (Config): connection config + field_name (Union[InterpolatedString, str]): the name of the field which will hold the cursor value for outbound API call + cursor_field (Union[InterpolatedString, str]): record's cursor field + """ + + config: Config + cursor_field: Union[InterpolatedString, str] + field_name: Union[InterpolatedString, str] + parameters: InitVar[Mapping[str, Any]] + _cursor: Optional[str] = field(repr=False, default=None) + message_repository: Optional[MessageRepository] = None + + def __post_init__(self, parameters: Mapping[str, Any]) -> None: + self._state: Optional[int] = None + self._interpolation = JinjaInterpolation() + self.cursor_field = InterpolatedString.create(self.cursor_field, parameters=parameters) + self.field_name = InterpolatedString.create(self.field_name, parameters=parameters) + + def get_stream_state(self) -> StreamState: + return {self.cursor_field.eval(self.config): self._cursor} if self._cursor else {} + + def set_initial_state(self, stream_state: StreamState) -> None: + """ + Cursors are not initialized with their state. As state is needed in order to function properly, this method should be called + before calling anything else + + :param stream_state: The state of the stream as returned by get_stream_state + """ + + self._cursor = stream_state.get(self.cursor_field.eval(self.config)) if stream_state else None + self._state = self._cursor if self._cursor else self._state + + def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None: + last_record_cursor_value = most_recent_record.get(self.cursor_field.eval(self.config)) if most_recent_record else None + self._cursor = last_record_cursor_value if last_record_cursor_value else None + + def stream_slices(self) -> Iterable[StreamSlice]: + """ + Use a single Slice. + """ + return [None] + + def get_request_params( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + return self._get_request_options(RequestOptionType.request_parameter, stream_slice) + + def _get_request_options(self, option_type: RequestOptionType, stream_slice: StreamSlice): + options = {} + if self._state: + options[self.field_name.eval(self.config)] = self._state + return options + + def should_be_synced(self, record: Record) -> bool: + cursor_field = self.cursor_field.eval(self.config) + record_cursor_value: int = record.get(cursor_field) + if not record_cursor_value: + self._send_log( + Level.WARN, + f"Could not find cursor field `{cursor_field}` in record. The incremental sync will assume it needs to be synced", + ) + return True + latest_possible_cursor_value = self._cursor if self._cursor else 0 + return latest_possible_cursor_value <= record_cursor_value + + def _send_log(self, level: Level, message: str) -> None: + if self.message_repository: + self.message_repository.emit_message( + AirbyteMessage( + type=Type.LOG, + log=AirbyteLogMessage(level=level, message=message), + ) + ) + + def is_greater_than_or_equal(self, first: Record, second: Record) -> bool: + cursor_field = self.cursor_field.eval(self.config) + first_cursor_value = first.get(cursor_field) + second_cursor_value = second.get(cursor_field) + if first_cursor_value and second_cursor_value: + return first_cursor_value >= second_cursor_value + elif first_cursor_value: + return True + else: + return False diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py new file mode 100644 index 0000000000000..ffc829461ef0a --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py @@ -0,0 +1,38 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import dataclass +from typing import Any, List, Mapping, Optional + +import requests +from airbyte_cdk.sources.declarative.requesters.paginators.strategies import OffsetIncrement + + +@dataclass +class IdOffsetIncrementPaginationStrategy(OffsetIncrement): + """ + Id Offset Pagination docs: + https://developer.zendesk.com/api-reference/live-chat/chat-api/agents/#pagination + + Attributes: + page_size (InterpolatedString): the number of records to request, + id_field (InterpolatedString): the name of the to track and increment from, {: 1234} + """ + + id_field: str = "id" + + def __post_init__(self, parameters: Mapping[str, Any], **kwargs): + self._id_field = self.id_field + super().__post_init__(parameters=parameters, **kwargs) + + def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: + decoded_response = self.decoder.decode(response) + # Stop paginating when there are fewer records than the page size or the current page has no records + if (self._page_size and len(last_records) < self._page_size.eval(self.config, response=decoded_response)) or len(last_records) == 0: + return None + else: + # the `IDs` are returned in `ASC` order, we add `+1` to the ID integer value to avoid the record duplicates, + # as described in: https://developer.zendesk.com/api-reference/live-chat/chat-api/agents/#pagination + self._offset = last_records[-1][self._id_field] + return self._offset + 1 diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml new file mode 100644 index 0000000000000..62643de5b356f --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml @@ -0,0 +1,273 @@ +version: 0.65.0 + +definitions: + # COMMON PARTS + schema_loader: + type: JsonFileSchemaLoader + file_path: "./source_zendesk_chat/schemas/{{ parameters['name'] }}.json" + selector: + description: >- + Base records selector for Full Refresh streams + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["{{ parameters.get('data_field') }}"] + authenticator: + type: BearerAuthenticator + api_token: "{{ config['credentials']['access_token'] }}" + + # PAGINATORS + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: cursor + page_size_option: + inject_into: request_parameter + type: RequestOption + field_name: limit + pagination_strategy: + type: CursorPagination + page_size: 2 + cursor_value: '{{ response.get("next_url", {}) }}' + stop_condition: '{{ not response.get("next_url", {}) }}' + paginator_id_offset: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: since_id + page_size_option: + inject_into: request_parameter + type: RequestOption + field_name: limit + pagination_strategy: + type: CustomPaginationStrategy + class_name: source_zendesk_chat.components.id_offset_pagination.IdOffsetIncrementPaginationStrategy + page_size: 100 + + # REQUESTERS + requester: + description: >- + Default Base Requester for Full Refresh streams + type: HttpRequester + url_base: https://www.zopim.com/api/v2/ + path: "{{ parameters['path'] }}" + http_method: GET + authenticator: + $ref: "#/definitions/authenticator" + error_handler: + type: DefaultErrorHandler + description: >- + The default error handler + # additional request kwargs + request_parameters: + timeout: "60" + + # RETRIEVERS + retriever_base: + description: >- + Default Retriever for Full Refresh streams + record_selector: + $ref: "#/definitions/selector" + requester: + $ref: "#/definitions/requester" + paginator: + $ref: "#/definitions/paginator" + retriever_for_type_list: + $ref: "#/definitions/retriever_base" + record_selector: + $ref: "#/definitions/selector" + extractor: + type: DpathExtractor + field_path: [] + retriever_for_type_list_no_pagination: + $ref: "#/definitions/retriever_for_type_list" + paginator: + type: NoPagination + + # BASE STREAMS + # FULL-REFRESH + base_stream: + primary_key: "id" + schema_loader: + $ref: "#/definitions/schema_loader" + retriever: + $ref: "#/definitions/retriever_base" + base_stream_with_list_response_no_pagination: + primary_key: "id" + schema_loader: + $ref: "#/definitions/schema_loader" + retriever: + $ref: "#/definitions/retriever_for_type_list_no_pagination" + base_stream_with_id_offset_pagination: + primary_key: "id" + schema_loader: + $ref: "#/definitions/schema_loader" + retriever: + $ref: "#/definitions/retriever_for_type_list" + paginator: + $ref: "#/definitions/paginator_id_offset" + + # INCREMENTAL + base_incremental_id_stream: + $ref: "#/definitions/base_stream_with_id_offset_pagination" + retriever: + $ref: "#/definitions/base_stream_with_id_offset_pagination/retriever" + # this is needed to ignore additional params for incremental syncs + ignore_stream_slicer_parameters_on_paginated_requests: true + incremental_sync: + type: CustomIncrementalSync + class_name: source_zendesk_chat.components.id_incremental.IdIncrementalCursor + cursor_field: "id" + field_name: "since_id" + + # FULL-REFRESH STREAMS + # ACCOUNTS + accounts_stream: + description: >- + Accounts Stream: https://developer.zendesk.com/rest_api/docs/chat/accounts#show-account + $ref: "#/definitions/base_stream_with_list_response_no_pagination" + primary_key: "account_key" + $parameters: + name: "accounts" + path: "account" + # SHORTCUTS + shortcuts_stream: + description: >- + Shortcuts Stream: https://developer.zendesk.com/rest_api/docs/chat/shortcuts#list-shortcuts + $ref: "#/definitions/base_stream_with_list_response_no_pagination" + $parameters: + name: "shortcuts" + path: "shortcuts" + # ROUTING SETTINGS + routing_settings_stream: + description: >- + Routing Settings Stream: https://developer.zendesk.com/rest_api/docs/chat/routing_settings#show-account-routing-settings + $ref: "#/definitions/base_stream" + primary_key: "" + $parameters: + name: "routing_settings" + data_field: "data" + path: "routing_settings/account" + # TRIGGERS + triggers_stream: + description: >- + Triggers Stream: https://developer.zendesk.com/rest_api/docs/chat/triggers#list-triggers + $ref: "#/definitions/base_stream_with_list_response_no_pagination" + $parameters: + name: "triggers" + path: "triggers" + # ROLES + roles_stream: + description: >- + Roles Stream: https://developer.zendesk.com/rest_api/docs/chat/roles#list-roles + $ref: "#/definitions/base_stream_with_list_response_no_pagination" + $parameters: + name: "roles" + path: "roles" + # SKILLS + skills_stream: + description: >- + Skills Stream: https://developer.zendesk.com/rest_api/docs/chat/skills#list-skills + $ref: "#/definitions/base_stream_with_list_response_no_pagination" + $parameters: + name: "skills" + path: "skills" + # GOALS + goals_stream: + description: >- + Goals Stream: https://developer.zendesk.com/rest_api/docs/chat/goals#list-goals + $ref: "#/definitions/base_stream_with_list_response_no_pagination" + $parameters: + name: "goals" + path: "goals" + # DEPARTMENTS + departments_stream: + description: >- + Departments Stream: https://developer.zendesk.com/rest_api/docs/chat/departments#list-departments + $ref: "#/definitions/base_stream_with_list_response_no_pagination" + $parameters: + name: "departments" + path: "departments" + + # COLLECTIONS + # collections_stream: + # description: >- + # Collections Stream: https://developer.rechargepayments.com/2021-11/collections/collections_list + # $ref: "#/definitions/base_modern_api_stream" + # $parameters: + # name: "collections" + # data_field: "collections" + # # METAFIELDS + # metafields_stream: + # description: >- + # Metafields Stream: https://developer.rechargepayments.com/2021-11/metafields + # $ref: "#/definitions/base_modern_api_stream" + # retriever: + # $ref: "#/definitions/retriever_metafields" + # $parameters: + # name: "metafields" + # data_field: "metafields" + # # PRODUCTS + # products_stream: + # description: >- + # Products Stream: https://developer.rechargepayments.com/2021-11/products/products_list + # Products endpoint has 422 error with 2021-11 API version + # $ref: "#/definitions/base_deprecated_api_stream" + # $parameters: + # name: "products" + # data_field: "products" + # # SHOP + # shop_stream: + # description: >- + # Shop Stream: https://developer.rechargepayments.com/v1-shopify?python#shop + # Shop endpoint is not available in 2021-11 API version + # $ref: "#/definitions/base_deprecated_api_stream" + # retriever: + # $ref: "#/definitions/retriever_api_deprecated" + # record_selector: + # $ref: "#/definitions/selector" + # extractor: + # type: DpathExtractor + # field_path: [] + # primary_key: ["shop", "store"] + # $parameters: + # name: "shop" + + # INCREMENTAL STREAMS + # AGENTS + agents_stream: + description: >- + Agents Stream: https://developer.zendesk.com/rest_api/docs/chat/agents#list-agents + $ref: "#/definitions/base_incremental_id_stream" + $parameters: + name: "agents" + path: "agents" + # BANS + bans_stream: + description: >- + Bans Stream: https://developer.zendesk.com/rest_api/docs/chat/bans#list-bans + $ref: "#/definitions/base_incremental_id_stream" + $parameters: + name: "bans" + path: "bans" + + +streams: + - "#/definitions/bans_stream" + - "#/definitions/agents_stream" + - "#/definitions/departments_stream" + - "#/definitions/goals_stream" + - "#/definitions/skills_stream" + - "#/definitions/roles_stream" + - "#/definitions/triggers_stream" + - "#/definitions/shortcuts_stream" + - "#/definitions/accounts_stream" + - "#/definitions/routing_settings_stream" + +check: + type: CheckStream + stream_names: + - routing_settings \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py index bcad700d13d4a..518a340254e2d 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py @@ -2,58 +2,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from typing import Any, Dict, List, Mapping, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from .streams import Accounts, Agents, AgentTimelines, Bans, Chats, Departments, Goals, Roles, RoutingSettings, Shortcuts, Skills, Triggers - - -class ZendeskAuthentication: - """Provides the authentication capabilities for both old and new methods.""" - - def __init__(self, config: Dict): - self.config = config - - def get_auth(self) -> TokenAuthenticator: - """Return the TokenAuthenticator object with access_token.""" - - # the old config supports for backward capability - access_token = self.config.get("access_token") - if not access_token: - # the new config supports `OAuth2.0` - access_token = self.config["credentials"]["access_token"] - - return TokenAuthenticator(token=access_token) - - -class SourceZendeskChat(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - authenticator = ZendeskAuthentication(config).get_auth() - try: - records = RoutingSettings(authenticator=authenticator).read_records(sync_mode=SyncMode.full_refresh) - next(records) - return True, None - except Exception as error: - return False, f"Unable to connect to Zendesk Chat API with the provided credentials - {error}" - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - authenticator = ZendeskAuthentication(config).get_auth() - return [ - Accounts(authenticator=authenticator), - AgentTimelines(authenticator=authenticator, start_date=config["start_date"]), - Agents(authenticator=authenticator), - Bans(authenticator=authenticator), - Chats(authenticator=authenticator, start_date=config["start_date"]), - Departments(authenticator=authenticator), - Goals(authenticator=authenticator), - Roles(authenticator=authenticator), - RoutingSettings(authenticator=authenticator), - Shortcuts(authenticator=authenticator), - Skills(authenticator=authenticator), - Triggers(authenticator=authenticator), - ] +# Declarative Source +class SourceZendeskChat(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) From c5b8fb064b14c25d9ee702342bb569f5376befaa Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 6 Mar 2024 11:23:21 +0200 Subject: [PATCH 04/23] added bans stream --- .../components/record_extractor.py | 27 +++++++++ .../source_zendesk_chat/manifest.yaml | 58 ++++--------------- 2 files changed, 38 insertions(+), 47 deletions(-) create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/record_extractor.py diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/record_extractor.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/record_extractor.py new file mode 100644 index 0000000000000..4d48c92b00fdf --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/record_extractor.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from dataclasses import dataclass +from typing import Any, List, Mapping + +import pendulum +import requests +from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor +from airbyte_cdk.sources.declarative.types import Record + + +@dataclass +class BansRecordExtractor(RecordExtractor): + """ + Unnesting nested bans: `visitor`, `ip_address`. + """ + + def extract_records(self, response: requests.Response) -> List[Record]: + response_data = response.json() + ip_address: List[Mapping[str, Any]] = response_data.get("ip_address", []) + visitor: List[Mapping[str, Any]] = response_data.get("visitor", []) + bans = ip_address + visitor + bans = sorted(bans, key=lambda x: pendulum.parse(x["created_at"]) if x["created_at"] else pendulum.datetime(1970, 1, 1)) + return bans diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml index 62643de5b356f..218f892d43cb1 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml @@ -12,6 +12,11 @@ definitions: extractor: type: DpathExtractor field_path: ["{{ parameters.get('data_field') }}"] + bans_stream_record_selector: + type: RecordSelector + extractor: + type: CustomRecordExtractor + class_name: source_zendesk_chat.components.record_extractor.BansRecordExtractor authenticator: type: BearerAuthenticator api_token: "{{ config['credentials']['access_token'] }}" @@ -29,7 +34,7 @@ definitions: field_name: limit pagination_strategy: type: CursorPagination - page_size: 2 + page_size: 100 cursor_value: '{{ response.get("next_url", {}) }}' stop_condition: '{{ not response.get("next_url", {}) }}' paginator_id_offset: @@ -85,7 +90,7 @@ definitions: retriever_for_type_list_no_pagination: $ref: "#/definitions/retriever_for_type_list" paginator: - type: NoPagination + type: NoPagination # BASE STREAMS # FULL-REFRESH @@ -192,50 +197,6 @@ definitions: name: "departments" path: "departments" - # COLLECTIONS - # collections_stream: - # description: >- - # Collections Stream: https://developer.rechargepayments.com/2021-11/collections/collections_list - # $ref: "#/definitions/base_modern_api_stream" - # $parameters: - # name: "collections" - # data_field: "collections" - # # METAFIELDS - # metafields_stream: - # description: >- - # Metafields Stream: https://developer.rechargepayments.com/2021-11/metafields - # $ref: "#/definitions/base_modern_api_stream" - # retriever: - # $ref: "#/definitions/retriever_metafields" - # $parameters: - # name: "metafields" - # data_field: "metafields" - # # PRODUCTS - # products_stream: - # description: >- - # Products Stream: https://developer.rechargepayments.com/2021-11/products/products_list - # Products endpoint has 422 error with 2021-11 API version - # $ref: "#/definitions/base_deprecated_api_stream" - # $parameters: - # name: "products" - # data_field: "products" - # # SHOP - # shop_stream: - # description: >- - # Shop Stream: https://developer.rechargepayments.com/v1-shopify?python#shop - # Shop endpoint is not available in 2021-11 API version - # $ref: "#/definitions/base_deprecated_api_stream" - # retriever: - # $ref: "#/definitions/retriever_api_deprecated" - # record_selector: - # $ref: "#/definitions/selector" - # extractor: - # type: DpathExtractor - # field_path: [] - # primary_key: ["shop", "store"] - # $parameters: - # name: "shop" - # INCREMENTAL STREAMS # AGENTS agents_stream: @@ -250,11 +211,14 @@ definitions: description: >- Bans Stream: https://developer.zendesk.com/rest_api/docs/chat/bans#list-bans $ref: "#/definitions/base_incremental_id_stream" + retriever: + $ref: "#/definitions/base_incremental_id_stream/retriever" + record_selector: + $ref: "#/definitions/bans_stream_record_selector" $parameters: name: "bans" path: "bans" - streams: - "#/definitions/bans_stream" - "#/definitions/agents_stream" From bb977e4626efcae46a3bf52f3862e2ffcda1920f Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 6 Mar 2024 17:35:04 +0200 Subject: [PATCH 05/23] updated --- .../acceptance-test-config.yml | 6 +- .../integration_tests/expected_records.jsonl | 32 ++++++ .../integration_tests/expected_records.txt | 34 ------ .../integration_tests/state.json | 14 +++ .../source-zendesk-chat/metadata.yaml | 3 +- .../components/datetime_based_cursor.py | 59 ++++++++++ .../components/id_incremental.py | 2 +- .../components/id_offset_pagination.py | 2 +- .../components/record_extractor.py | 2 +- .../components/time_offset_pagination.py | 41 +++++++ .../source_zendesk_chat/manifest.yaml | 104 ++++++++++++++++-- .../schemas/agent_timeline.json | 3 + 12 files changed, 248 insertions(+), 54 deletions(-) create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.jsonl delete mode 100644 airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.txt create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/integration_tests/state.json create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/datetime_based_cursor.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py diff --git a/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml index b0acaaaefee0a..674c07534f860 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml @@ -6,8 +6,6 @@ acceptance_tests: - spec_path: "source_zendesk_chat/spec.json" connection: tests: - - config_path: "secrets/config_old.json" - status: "succeed" - config_path: "secrets/config.json" status: "succeed" - config_path: "secrets/config_oauth.json" @@ -23,10 +21,10 @@ acceptance_tests: tests: - config_path: "secrets/config.json" expect_records: - path: "integration_tests/expected_records.txt" + path: "integration_tests/expected_records.jsonl" - config_path: "secrets/config_oauth.json" expect_records: - path: "integration_tests/expected_records.txt" + path: "integration_tests/expected_records.jsonl" incremental: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.jsonl new file mode 100644 index 0000000000000..726145d3a07fb --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.jsonl @@ -0,0 +1,32 @@ +{"stream": "accounts", "data": {"create_date": "2020-12-11T18:33:40Z", "account_key": "svBRNv6HoJnSZRgpf6yMmBZaFMY6s2hP", "status": "active", "plan": {"goals": 5, "long_desc": "Best for Organizations to manage Large Support Teams", "price": 70.0, "short_desc": "Ideal for Large Support Teams", "widget_customization": "full", "max_agents": 5, "sla": true, "monitoring": true, "rest_api": true, "email_reports": true, "daily_reports": true, "chat_reports": true, "agent_reports": true, "agent_leaderboard": true, "unbranding": true, "high_load": true, "ip_restriction": true, "support": true, "name": "enterprise", "max_basic_triggers": "unlimited", "max_advanced_triggers": "unlimited", "max_departments": "unlimited", "max_concurrent_chats": "unlimited", "max_history_search_days": "unlimited", "operating_hours": true, "file_upload": true, "analytics": true, "integrations": true}}, "emitted_at": 1709738612256} +{"stream": "agents", "data": {"enabled": true, "create_date": "2020-11-17T23:55:24Z", "role_id": 360002848996, "first_name": "Team Airbyte", "email": "integration-test@airbyte.io", "last_name": "", "id": 360786799676, "enabled_departments": [7282618889231], "departments": [7282618889231, 5059474192015, 5060105343503, 5060005480847, 5060049125391, 5060061403535, 5060061508879, 5060049288719, 5060049443215, 5060066676751, 5060066798607, 5060071902479, 5060093166863, 5060100872591, 5060101239823, 5060072765583, 5060101350159, 5060077702799, 5060088742799, 5060103345935, 5060078913935, 5059463979663, 5060103664783, 5060079026575, 5060055796111, 5060090959759, 5059403825935, 5060108375311, 5059473809295, 5059436114575, 360003074836, 6770788212111], "skills": [], "display_name": "Team Airbyte", "last_login": "2024-02-09T13:12:16Z", "login_count": 113, "roles": {"administrator": true, "owner": false}}, "emitted_at": 1709738612909} +{"stream": "agents", "data": {"enabled": true, "create_date": "2021-04-23T14:33:11Z", "role_id": 360002848976, "first_name": "Fake User number - 1", "email": "fake.user-1@email.com", "last_name": "", "id": 361084605116, "enabled_departments": [7282640316815, 7282630247567, 7282624630287], "departments": [7282640316815, 7282630247567, 7282624630287, 5060105343503, 5060005480847, 5060049125391, 5060061403535, 5060061508879, 5060049288719, 5060049443215, 5060066676751, 5060066798607, 5060071902479, 5059452990735, 5060093166863, 5060100872591, 5060101239823, 5060072765583, 5060101350159, 5060077702799, 5060088742799, 5060103345935, 5060078913935, 5060103664783, 5060079026575, 5060055796111, 5060090959759, 5059473603087, 5059403825935, 5060108375311, 5059473809295, 5059436284943, 360003074836], "skills": [1300601, 8565161], "display_name": "Fake User number - 1", "last_login": null, "login_count": 0, "roles": {"administrator": false, "owner": false}}, "emitted_at": 1709738612913} +{"stream": "agents", "data": {"enabled": true, "create_date": "2021-04-23T14:34:20Z", "role_id": 360002848976, "first_name": "Fake Agent number - 1", "email": "fake.agent-1@email.com", "last_name": "", "id": 361089721035, "enabled_departments": [7282630247567], "departments": [7282630247567, 7282657193103, 5059439464079, 5060105343503, 5060005480847, 5060049125391, 5060061403535, 5060061508879, 5060049288719, 5060049443215, 5060066676751, 5060066798607, 5060071902479, 5060093166863, 5060100872591, 5060101239823, 5060072765583, 5060101350159, 5060077702799, 5060088742799, 5060103345935, 5060078913935, 5060103664783, 5060079026575, 5060055796111, 5060090959759, 5059473603087, 5060108375311, 5059473809295, 5059436114575, 5059404003599, 360003074836], "skills": [1296081, 1300641], "display_name": "Fake Agent number - 1", "last_login": null, "login_count": 0, "roles": {"administrator": false, "owner": false}}, "emitted_at": 1709738612916} +{"stream": "agent_timeline", "data": {"agent_id": 360786799676, "engagement_count": 0, "start_time": "2020-12-14T04:08:32.301292Z", "status": "invisible", "duration": 459.213926, "id": "360786799676|2020-12-14T04:08:32.301292Z"}, "emitted_at": 1709738613859} +{"stream": "agent_timeline", "data": {"agent_id": 360786799676, "engagement_count": 0, "start_time": "2020-12-14T04:17:32.387364Z", "status": "invisible", "duration": 3440.710507, "id": "360786799676|2020-12-14T04:17:32.387364Z"}, "emitted_at": 1709738613863} +{"stream": "agent_timeline", "data": {"agent_id": 360786799676, "engagement_count": 0, "start_time": "2020-12-14T18:45:37.160254Z", "status": "invisible", "duration": 520.75554, "id": "360786799676|2020-12-14T18:45:37.160254Z"}, "emitted_at": 1709738613864} +{"stream": "bans", "data": {"created_at": "2021-04-21T14:42:46Z", "reason": "Spammer", "type": "ip_address", "id": 70519881, "ip_address": "192.123.123.5"}, "emitted_at": 1709738615366} +{"stream": "bans", "data": {"created_at": "2021-04-26T13:55:20Z", "reason": "Spammer", "type": "ip_address", "id": 75112241, "ip_address": "191.121.123.5"}, "emitted_at": 1709738615369} +{"stream": "bans", "data": {"created_at": "2021-04-26T13:55:30Z", "reason": "Spammer", "type": "ip_address", "id": 75112281, "ip_address": "111.121.123.5"}, "emitted_at": 1709738615371} +{"stream": "chats", "data": {"timestamp": "2021-04-26T13:54:02Z", "unread": false, "webpath": [], "type": "offline_msg", "id": "2104.10414779.SVhDCJ9flq79a", "update_timestamp": "2021-04-27T15:09:17Z", "tags": [], "department_name": null, "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2014-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "department_id": null, "deleted": false, "message": "Hi there!", "visitor": {"phone": "+32178763521", "notes": "Test 2", "id": "3.45678", "name": "Jiny", "email": "visitor_jiny@doe.com"}, "zendesk_ticket_id": null}, "emitted_at": 1709738618587} +{"stream": "chats", "data": {"timestamp": "2021-04-21T14:36:55Z", "unread": false, "webpath": [], "type": "offline_msg", "id": "2104.10414779.SVE9Mo9bE4wR8", "update_timestamp": "2021-04-30T11:06:19Z", "tags": [], "department_name": null, "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2014-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "department_id": null, "deleted": false, "message": "Hi there!", "visitor": {"phone": "", "notes": "", "id": "1.12345", "name": "John", "email": "visitor_john@doe.com"}, "zendesk_ticket_id": null}, "emitted_at": 1709738618592} +{"stream": "chats", "data": {"timestamp": "2021-04-26T13:53:30Z", "unread": false, "webpath": [], "type": "offline_msg", "id": "2104.10414779.SVhD3v7I1LBOq", "update_timestamp": "2021-04-30T11:08:12Z", "tags": [], "department_name": null, "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2014-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "department_id": null, "deleted": false, "message": "Hi there!", "visitor": {"phone": "+78763521", "notes": "Test", "id": "2.34567", "name": "Tiny", "email": "visitor_tiny@doe.com"}, "zendesk_ticket_id": null}, "emitted_at": 1709738618596} +{"stream": "departments", "data": {"name": "Airbyte Department 1", "members": [361084605116], "settings": {"chat_enabled": true, "solved_ticket_reassignment_strategy": "", "support_group_id": 7282640316815}, "description": "A sample department", "id": 7282640316815, "enabled": true}, "emitted_at": 1709738620228} +{"stream": "departments", "data": {"name": "Department 1", "members": [360786799676], "settings": {"chat_enabled": true, "solved_ticket_reassignment_strategy": "", "support_group_id": 7282618889231}, "description": "A sample department", "id": 7282618889231, "enabled": true}, "emitted_at": 1709738620231} +{"stream": "departments", "data": {"name": "Department 2", "members": [361089721035, 361084605116], "settings": {"chat_enabled": true, "solved_ticket_reassignment_strategy": "", "support_group_id": 7282630247567}, "description": "A sample department 2", "id": 7282630247567, "enabled": true}, "emitted_at": 1709738620233} +{"stream": "goals", "data": {"enabled": true, "attribution_model": "first_touch", "description": "A new goal", "name": "Goal 3", "id": 513481, "attribution_period": 15, "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1709738621059} +{"stream": "goals", "data": {"enabled": false, "attribution_model": "first_touch", "description": "A new goal - 1", "name": "Goal one", "id": 529641, "attribution_period": 5, "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1709738621061} +{"stream": "goals", "data": {"enabled": false, "attribution_model": "first_touch", "description": "A new goal - 2", "name": "Goal two", "id": 529681, "attribution_period": 15, "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1709738621063} +{"stream": "roles", "data": {"enabled": true, "permissions": {"visitors_seen": "account", "proactive_chatting": "listen-join", "edit_visitor_information": true, "edit_visitor_notes": true, "view_past_chats": "account", "edit_chat_tags": true, "manage_bans": "account", "access_analytics": "account", "view_monitor": "account", "edit_department_agents": "account", "set_agent_chat_limit": "account", "manage_shortcuts": "account"}, "description": "In addition to regular agent privileges, administrators can edit widget and accounts settings, manage agents, roles and permissions, and more. Permissions for the administrator role cannot be modified.", "name": "Administrator", "id": 360002848996, "members_count": 1}, "emitted_at": 1709738621711} +{"stream": "roles", "data": {"enabled": true, "permissions": {"visitors_seen": "account", "proactive_chatting": "listen-join", "edit_visitor_information": true, "edit_visitor_notes": true, "view_past_chats": "account", "edit_chat_tags": false, "manage_bans": "account", "access_analytics": "none", "view_monitor": "account", "edit_department_agents": "none", "set_agent_chat_limit": "none", "manage_shortcuts": "account"}, "description": "Agent is the most basic role in an account, and their primary responsibility is to serve chats. Permissions for the agent role can be modified.", "name": "Agent", "id": 360002848976, "members_count": 2}, "emitted_at": 1709738621714} +{"stream": "roles", "data": {"enabled": true, "permissions": {"visitors_seen": "account", "proactive_chatting": "listen-join", "edit_visitor_information": true, "edit_visitor_notes": true, "view_past_chats": "account", "edit_chat_tags": false, "manage_bans": "account", "access_analytics": "none", "view_monitor": "account", "edit_department_agents": "none", "set_agent_chat_limit": "none", "manage_shortcuts": "account"}, "description": "Can serve social messaging conversations only", "name": "Agent (limited)", "id": 7282769201935, "members_count": 0}, "emitted_at": 1709738621715} +{"stream": "routing_settings", "data": {"routing_mode": "assigned", "chat_limit": {"enabled": false, "limit": 3, "limit_type": "account", "allow_agent_override": false}, "skill_routing": {"enabled": true, "max_wait_time": 30}, "reassignment": {"enabled": true, "timeout": 30}, "auto_idle": {"enabled": false, "reassignments_before_idle": 3, "new_status": "away"}, "auto_accept": {"enabled": false}}, "emitted_at": 1709738622442} +{"stream": "shortcuts", "data": {"options": "Yes/No", "id": "goodbye", "scope": "all", "name": "goodbye", "tags": ["goodbye_survey"], "message": "Thanks for chatting with us. Have we resolved your question(s)?"}, "emitted_at": 1709738623130} +{"stream": "shortcuts", "data": {"options": "Yes/No", "id": "help", "scope": "all", "name": "help", "tags": ["help_survey"], "message": "Do you need any help?"}, "emitted_at": 1709738623133} +{"stream": "shortcuts", "data": {"options": "", "id": "hi", "scope": "all", "name": "hi", "tags": [], "message": "Hi, how can we help you today? =)"}, "emitted_at": 1709738623135} +{"stream": "skills", "data": {"id": 1300601, "name": "english", "members": [361084605116], "description": "English language", "enabled": true}, "emitted_at": 1709738623775} +{"stream": "skills", "data": {"id": 1300641, "name": "france", "members": [361089721035], "description": "France language", "enabled": true}, "emitted_at": 1709738623778} +{"stream": "skills", "data": {"id": 1296081, "name": "mandarin", "members": [361089721035], "description": "Chinese language", "enabled": true}, "emitted_at": 1709738623780} +{"stream": "triggers", "data": {"enabled": true, "definition": {"condition": ["and", ["not", ["firedBefore"]], ["and", ["neq", "@account_status", "offline"], ["stillOnSite", 60], ["eq", "@visitor_served", false]]], "event": "chat_requested", "actions": [["setTriggered", true], ["sendMessageToVisitor", "Customer Service", "We apologize for keeping you waiting. Our operators are busy at the moment, please leave us a message with your email address and we'll get back to you shortly."]]}, "description": "Auto respond to messages if agents don't respond in time.", "name": "Chat Rescuer", "id": 66052481}, "emitted_at": 1709738624471} +{"stream": "triggers", "data": {"enabled": true, "definition": {"event": "page_enter", "condition": ["and", ["eq", "@visitor_page_url", "www.zendesk.com/cart"], ["stillOnPage", 60], ["eq", "@visitor_requesting_chat", false], ["eq", "@visitor_served", false], ["not", ["firedBefore"]]], "actions": [["sendMessageToVisitor", "Stephanie", "Hi, are you having any trouble checking out? Feel free to reach out to us with any questions."]], "version": 1, "editor": "advanced"}, "description": "Reduce cart abandonment by engaging customers that are lingering on the checkout page.", "name": "Checkout Page", "id": 66052561}, "emitted_at": 1709738624474} +{"stream": "triggers", "data": {"enabled": true, "definition": {"event": "chat_requested", "condition": ["and", ["eq", "@visitor_requesting_chat", true], ["eq", "@visitor_served", false], ["not", ["firedBefore"]]], "actions": [["wait", 5], ["sendMessageToVisitor", "Customer Service", "Thanks for your message, please wait a moment while our agents attend to you."]], "version": 1, "editor": "advanced"}, "description": "Send an automated reply to customers that start a chat, so they know their request is being attended to.", "name": "First Reply", "id": 66052601}, "emitted_at": 1709738624476} diff --git a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.txt deleted file mode 100644 index 10f75ba0af99f..0000000000000 --- a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.txt +++ /dev/null @@ -1,34 +0,0 @@ -{"stream": "accounts", "data": {"create_date": "2020-12-11T18:33:40Z", "status": "active", "account_key": "svBRNv6HoJnSZRgpf6yMmBZaFMY6s2hP", "plan": {"goals": 5, "long_desc": "Best for Organizations to manage Large Support Teams", "price": 70.0, "short_desc": "Ideal for Large Support Teams", "widget_customization": "full", "max_agents": 5, "sla": true, "monitoring": true, "rest_api": true, "email_reports": true, "daily_reports": true, "chat_reports": true, "agent_reports": true, "agent_leaderboard": true, "unbranding": true, "high_load": true, "ip_restriction": true, "support": true, "name": "enterprise", "max_basic_triggers": "unlimited", "max_advanced_triggers": "unlimited", "max_departments": "unlimited", "max_concurrent_chats": "unlimited", "max_history_search_days": "unlimited", "operating_hours": true, "file_upload": true, "analytics": true, "integrations": true}}, "emitted_at": 1672828432816} -{"stream": "agent_timeline", "data": {"agent_id": 360786799676, "engagement_count": 0, "start_time": "2022-01-17T13:20:50Z", "status": "invisible", "duration": 789.733983, "id": "360786799676|2022-01-17T13:20:50Z"}, "emitted_at": 1672828433249} -{"stream": "agent_timeline", "data": {"agent_id": 360786799676, "engagement_count": 0, "start_time": "2022-06-30T17:16:55Z", "status": "invisible", "duration": 61.089883, "id": "360786799676|2022-06-30T17:16:55Z"}, "emitted_at": 1672828433249} -{"stream": "agent_timeline", "data": {"agent_id": 360786799676, "engagement_count": 0, "start_time": "2022-10-28T12:43:05Z", "status": "invisible", "duration": 370.793077, "id": "360786799676|2022-10-28T12:43:05Z"}, "emitted_at": 1672828433249} -{"stream": "agents", "data": {"role_id": 360002848976, "departments": [7282640316815, 7282630247567, 7282624630287, 5060105343503, 5060005480847, 5060049125391, 5060061403535, 5060061508879, 5060049288719, 5060049443215, 5060066676751, 5060066798607, 5060071902479, 5059452990735, 5060093166863, 5060100872591, 5060101239823, 5060072765583, 5060101350159, 5060077702799, 5060088742799, 5060103345935, 5060078913935, 5060103664783, 5060079026575, 5060055796111, 5060090959759, 5059473603087, 5059403825935, 5060108375311, 5059473809295, 5059436284943, 360003074836], "enabled_departments": [7282640316815, 7282630247567, 7282624630287], "last_name": "", "create_date": "2021-04-23T14:33:11Z", "first_name": "Fake User number - 1", "enabled": true, "skills": [1300601, 8565161], "id": 361084605116, "display_name": "Fake User number - 1", "email": "fake.user-1@email.com", "last_login": null, "login_count": 0, "roles": {"administrator": false, "owner": false}}, "emitted_at": 1688547518353} -{"stream": "agents", "data": {"role_id": 360002848976, "departments": [7282630247567, 7282657193103, 5059439464079, 5060105343503, 5060005480847, 5060049125391, 5060061403535, 5060061508879, 5060049288719, 5060049443215, 5060066676751, 5060066798607, 5060071902479, 5060093166863, 5060100872591, 5060101239823, 5060072765583, 5060101350159, 5060077702799, 5060088742799, 5060103345935, 5060078913935, 5060103664783, 5060079026575, 5060055796111, 5060090959759, 5059473603087, 5060108375311, 5059473809295, 5059436114575, 5059404003599, 360003074836], "enabled_departments": [7282630247567], "last_name": "", "create_date": "2021-04-23T14:34:20Z", "first_name": "Fake Agent number - 1", "enabled": true, "skills": [1296081, 1300641], "id": 361089721035, "display_name": "Fake Agent number - 1", "email": "fake.agent-1@email.com", "last_login": null, "login_count": 0, "roles": {"administrator": false, "owner": false}}, "emitted_at": 1688547518353} -{"stream": "bans", "data": {"type": "visitor", "id": 75411361, "reason": "Spammer", "created_at": "2021-04-27T15:52:32Z", "visitor_name": "Visitor 47225177", "visitor_id": "10414779.13ojzHu7ISdt0SM"}, "emitted_at": 1672828433831} -{"stream": "bans", "data": {"type": "visitor", "id": 75411401, "reason": "Spammer", "created_at": "2021-04-27T15:52:32Z", "visitor_name": "Visitor 62959049", "visitor_id": "10414779.13ojzHu7at4VKcG"}, "emitted_at": 1672828433831} -{"stream": "bans", "data": {"created_at": "2021-04-27T15:52:32Z", "visitor_id": "10414779.13ojzHu7at4VKcG", "id": 75411401, "reason": "Spammer", "visitor_name": "Visitor 62959049", "type": "visitor"}, "emitted_at": 1672828434000} -{"stream": "bans", "data": {"created_at": "2021-04-27T15:52:33Z", "visitor_id": "10414779.13ojzHu7s9YwIjz", "id": 75411441, "reason": "Spammer", "visitor_name": "Visitor 97350211", "type": "visitor"}, "emitted_at": 1672828434001} -{"stream": "chats", "data": {"department_id": null, "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2014-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "visitor": {"phone": "+32178763521", "notes": "Test 2", "id": "3.45678", "name": "Jiny", "email": "visitor_jiny@doe.com"}, "update_timestamp": "2021-04-27T15:09:17Z", "department_name": null, "type": "offline_msg", "deleted": false, "tags": [], "timestamp": "2021-04-26T13:54:02Z", "unread": false, "id": "2104.10414779.SVhDCJ9flq79a", "message": "Hi there!", "zendesk_ticket_id": null}, "emitted_at": 1701452730189} -{"stream": "chats", "data": {"department_id": null, "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2014-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "visitor": {"phone": "", "notes": "", "id": "1.12345", "name": "John", "email": "visitor_john@doe.com"}, "update_timestamp": "2021-04-30T11:06:19Z", "department_name": null, "type": "offline_msg", "deleted": false, "tags": [], "timestamp": "2021-04-21T14:36:55Z", "unread": false, "id": "2104.10414779.SVE9Mo9bE4wR8", "message": "Hi there!", "zendesk_ticket_id": null}, "emitted_at": 1701452730190} -{"stream": "chats", "data": {"department_id": null, "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2014-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "visitor": {"phone": "+78763521", "notes": "Test", "id": "2.34567", "name": "Tiny", "email": "visitor_tiny@doe.com"}, "update_timestamp": "2021-04-30T11:08:12Z", "department_name": null, "type": "offline_msg", "deleted": false, "tags": [], "timestamp": "2021-04-26T13:53:30Z", "unread": false, "id": "2104.10414779.SVhD3v7I1LBOq", "message": "Hi there!", "zendesk_ticket_id": null}, "emitted_at": 1701452730190} -{"stream": "chats", "data": {"department_id": null, "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2022-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "visitor": {"phone": "", "notes": "", "id": "7.34502", "name": "Fake user - chat 2", "email": "fake_user_chat_2@doe.com"}, "update_timestamp": "2021-04-30T13:32:27Z", "department_name": null, "type": "offline_msg", "deleted": false, "tags": [], "timestamp": "2021-04-30T13:32:27Z", "unread": true, "id": "2104.10414779.SW4VrjJpOq6gk", "message": "Hi there!", "zendesk_ticket_id": null}, "emitted_at": 1701452730191} -{"stream": "departments", "data": {"settings": {"chat_enabled": true, "support_group_id": 7282640316815}, "members": [361084605116], "name": "Airbyte Department 1", "enabled": true, "description": "A sample department", "id": 7282640316815}, "emitted_at": 1688547521914} -{"stream": "departments", "data": {"settings": {"chat_enabled": true, "support_group_id": 7282618889231}, "members": [360786799676], "name": "Department 1", "enabled": true, "description": "A sample department", "id": 7282618889231}, "emitted_at": 1688547521914} -{"stream": "departments", "data": {"settings": {"chat_enabled": true, "support_group_id": 7282630247567}, "members": [361089721035, 361084605116], "name": "Department 2", "enabled": true, "description": "A sample department 2", "id": 7282630247567}, "emitted_at": 1688547521914} -{"stream": "goals", "data": {"enabled": true, "id": 513481, "attribution_period": 15, "attribution_model": "first_touch", "name": "Goal 3", "description": "A new goal", "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1701453031915} -{"stream": "goals", "data": {"enabled": false, "id": 529641, "attribution_period": 5, "attribution_model": "first_touch", "name": "Goal one", "description": "A new goal - 1", "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1701453031916} -{"stream": "goals", "data": {"enabled": false, "id": 529681, "attribution_period": 15, "attribution_model": "first_touch", "name": "Goal two", "description": "A new goal - 2", "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1701453031916} -{"stream": "goals", "data": {"enabled": true, "id": 537121, "attribution_period": 30, "attribution_model": "last_touch", "name": "Test goal", "description": "Test goal", "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://zendesk.com/thanks"}]}}, "emitted_at": 1701453031916} -{"stream": "roles", "data": {"permissions": {"visitors_seen": "account", "proactive_chatting": "listen-join", "edit_visitor_information": true, "edit_visitor_notes": true, "view_past_chats": "account", "edit_chat_tags": true, "manage_bans": "account", "access_analytics": "account", "view_monitor": "account", "edit_department_agents": "account", "set_agent_chat_limit": "account", "manage_shortcuts": "account"}, "enabled": true, "description": "In addition to regular agent privileges, administrators can edit widget and accounts settings, manage agents, roles and permissions, and more. Permissions for the administrator role cannot be modified.", "id": 360002848996, "name": "Administrator", "members_count": 1}, "emitted_at": 1672828435141} -{"stream": "roles", "data": {"permissions": {"visitors_seen": "account", "proactive_chatting": "listen-join", "edit_visitor_information": true, "edit_visitor_notes": true, "view_past_chats": "account", "edit_chat_tags": false, "manage_bans": "account", "access_analytics": "none", "view_monitor": "account", "edit_department_agents": "none", "set_agent_chat_limit": "none", "manage_shortcuts": "account"}, "enabled": true, "description": "Agent is the most basic role in an account, and their primary responsibility is to serve chats. Permissions for the agent role can be modified.", "id": 360002848976, "name": "Agent", "members_count": 2}, "emitted_at": 1672828435142} -{"stream": "shortcuts", "data": {"name": "goodbye", "id": "goodbye", "options": "Yes/No", "tags": ["goodbye_survey"], "scope": "all", "message": "Thanks for chatting with us. Have we resolved your question(s)?"}, "emitted_at": 1672828435386} -{"stream": "shortcuts", "data": {"name": "help", "id": "help", "options": "Yes/No", "tags": ["help_survey"], "scope": "all", "message": "Do you need any help?"}, "emitted_at": 1672828435386} -{"stream": "shortcuts", "data": {"name": "hi", "id": "hi", "options": "", "tags": [], "scope": "all", "message": "Hi, how can we help you today? =)"}, "emitted_at": 1672828435386} -{"stream": "shortcuts", "data": {"name": "returning", "id": "returning", "options": "", "tags": ["returning_visitor"], "scope": "all", "message": "Welcome back. How can we help you today"}, "emitted_at": 1672828435387} -{"stream": "skills", "data": {"id": 1300601, "name": "english", "enabled": true, "description": "English language", "members": [361084605116]}, "emitted_at": 1672828435627} -{"stream": "skills", "data": {"id": 1300641, "name": "france", "enabled": true, "description": "France language", "members": [361089721035]}, "emitted_at": 1672828435628} -{"stream": "skills", "data": {"id": 1296081, "name": "mandarin", "enabled": true, "description": "Chinese language", "members": [361089721035]}, "emitted_at": 1672828435628} -{"stream": "triggers", "data": {"name": "Product Discounts", "enabled": true, "description": "Offer your returning customers a discount on one of your products or services. This Trigger will need to be customized based on the page.", "id": 66052801, "definition": {"event": "chat_requested", "condition": ["and", ["icontains", "@visitor_page_url", "[product name]"], ["stillOnPage", 30], ["eq", "@visitor_requesting_chat", false], ["eq", "@visitor_served", false], ["not", ["firedBefore"]]], "actions": [["sendMessageToVisitor", "Customer Service", "Hi, are you interested in [insert product name]? We're offering a one-time 20% discount. Chat with me to find out more."]], "version": 1, "editor": "advanced"}}, "emitted_at": 1688547525543} -{"stream": "triggers", "data": {"name": "Request Contact Details", "enabled": true, "description": "When your account is set to away, ask customer's requesting a chat to leave their email address.", "id": 66052841, "definition": {"event": "chat_requested", "condition": ["and", ["eq", "@account_status", "away"], ["not", ["firedBefore"]]], "actions": [["addTag", "Away_request"], ["sendMessageToVisitor", "Customer Service", "Hi, sorry we are away at the moment. Please leave your email address and we will get back to you as soon as possible."]], "version": 1, "editor": "advanced"}}, "emitted_at": 1688547525543} -{"stream": "triggers", "data": {"name": "Tag Repeat Visitors", "enabled": true, "description": "Add a tag to a visitor that has visited your site 5 or more times. This helps you identify potential customers who are very interested in your brand.", "id": 66052881, "definition": {"event": "page_enter", "condition": ["and", ["gte", "@visitor_previous_visits", 5]], "actions": [["addTag", "5times"]], "version": 1, "editor": "advanced"}}, "emitted_at": 1688547525543} -{"stream": "routing_settings", "data": {"routing_mode": "assigned", "chat_limit": {"enabled": false, "limit": 3, "limit_type": "account", "allow_agent_override": false}, "skill_routing": {"enabled": true, "max_wait_time": 30}, "reassignment": {"enabled": true, "timeout": 30}, "auto_idle": {"enabled": false, "reassignments_before_idle": 3, "new_status": "away"}, "auto_accept": {"enabled": false}}, "emitted_at": 1701453336379} diff --git a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/state.json b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/state.json new file mode 100644 index 0000000000000..43873ad918475 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/state.json @@ -0,0 +1,14 @@ +{ + "agents": { + "id": 361089721035 + }, + "bans": { + "id": 75412441 + }, + "chats": { + "update_timestamp": "2023-10-20T09:44:12Z" + }, + "agent_timeline": { + "start_time": "2024-02-09T13:12:16.817973Z" + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml index c3e9a65b45168..9896737c16445 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml @@ -31,7 +31,8 @@ data: breakingChanges: 1.0.0: message: >- - This is the upgrade! Wow! + This update migrates the `source-zendesk-chat` to `YamlDeclarativeSource (Low-code)`. + More info in this PR: ____ upgradeDeadline: "2024-04-10" supportLevel: certified tags: diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/datetime_based_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/datetime_based_cursor.py new file mode 100644 index 0000000000000..7aa9ec4a160b7 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/datetime_based_cursor.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from dataclasses import dataclass +from typing import Any, Mapping, MutableMapping, Optional, Union + +from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor +from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString + + +@dataclass +class ZendeskChatTimestampCursor(DatetimeBasedCursor): + """ + Override for the default `DatetimeBasedCursor` to make self.close_slice() to produce the STATE from the RECORD cursor value instead of slicer values. + The dates in future are not allowed for the Zendesk Chat endpoints, and slicer values could be far away from exact cursor values. + + Arguments: + use_microseconds: bool - whether or not to add dummy `000000` (six zeros) to provide the microseconds unit timestamps + """ + + use_microseconds: Union[InterpolatedString, str] = True + + def __post_init__(self, parameters: Mapping[str, Any]) -> None: + self._use_microseconds = InterpolatedString.create(self.use_microseconds, parameters=parameters).eval(self.config) + self._start_date = self.config.get("start_date") + super().__post_init__(parameters=parameters) + + def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None: + last_record_cursor_value = most_recent_record.get(self.cursor_field.eval(self.config)) if most_recent_record else None + self._cursor = last_record_cursor_value if last_record_cursor_value else self._start_date + + def add_microseconds( + self, + field_name: str, + params: MutableMapping[str, Any], + stream_slice: Optional[StreamSlice] = None, + ) -> MutableMapping[str, Any]: + start_time = stream_slice.get("start_time") + if start_time: + params["start_time"] = int(start_time) * 1000000 + return params + + + def get_request_params( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + params = {} + if self._use_microseconds: + params = self.add_microseconds("start_time", params, stream_slice) + else: + params['start_time'] = stream_slice.get("start_time") + return params diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental.py index ead6bd205cc27..f2939f0679116 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental.py @@ -15,7 +15,7 @@ @dataclass -class IdIncrementalCursor(Cursor): +class ZendeskChatIdIncrementalCursor(Cursor): """ Custom Incremental Cursor implementation to provide the ability to pull data using `id`(int) as cursor. More info: https://developer.zendesk.com/api-reference/live-chat/chat-api/agents/#parameters diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py index ffc829461ef0a..0f2ead694b4ef 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py @@ -10,7 +10,7 @@ @dataclass -class IdOffsetIncrementPaginationStrategy(OffsetIncrement): +class ZendeskChatIdOffsetIncrementPaginationStrategy(OffsetIncrement): """ Id Offset Pagination docs: https://developer.zendesk.com/api-reference/live-chat/chat-api/agents/#pagination diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/record_extractor.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/record_extractor.py index 4d48c92b00fdf..688e892baaa60 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/record_extractor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/record_extractor.py @@ -13,7 +13,7 @@ @dataclass -class BansRecordExtractor(RecordExtractor): +class ZendeskChatBansRecordExtractor(RecordExtractor): """ Unnesting nested bans: `visitor`, `ip_address`. """ diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py new file mode 100644 index 0000000000000..cec46aaae078d --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py @@ -0,0 +1,41 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import dataclass +from typing import Any, List, Mapping, Optional + +import requests +from airbyte_cdk.sources.declarative.requesters.paginators.strategies import OffsetIncrement + + +@dataclass +class ZendeskChatTimeOffsetIncrementPaginationStrategy(OffsetIncrement): + """ + Time Offset Pagination docs: + https://developer.zendesk.com/api-reference/live-chat/chat-api/agents/#pagination + + Attributes: + page_size (InterpolatedString): the number of records to request, + time_field_name (InterpolatedString): the name of the to track and increment from, {: 1234} + """ + + time_field_name: str = None + + def __post_init__(self, parameters: Mapping[str, Any], **kwargs): + if not self.time_field_name: + raise Exception("The `time_field_name` property is missing, with no-default value.") + else: + self._time_field_name = self.time_field_name + super().__post_init__(parameters=parameters, **kwargs) + + def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: + decoded_response = self.decoder.decode(response) + # Stop paginating when there are fewer records than the page size or the current page has no records + if (self._page_size and len(last_records) < self._page_size.eval(self.config, response=decoded_response)) or len(last_records) == 0: + return None + else: + # the `records` are returned in `ASC` order, + # as described in: https://developer.zendesk.com/api-reference/live-chat/chat-api/incremental_export/#incremental-agent-timeline-export + self._offset = decoded_response[self._time_field_name] + return self._offset diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml index 218f892d43cb1..11895716a1b38 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml @@ -16,7 +16,7 @@ definitions: type: RecordSelector extractor: type: CustomRecordExtractor - class_name: source_zendesk_chat.components.record_extractor.BansRecordExtractor + class_name: source_zendesk_chat.components.record_extractor.ZendeskChatBansRecordExtractor authenticator: type: BearerAuthenticator api_token: "{{ config['credentials']['access_token'] }}" @@ -49,9 +49,24 @@ definitions: field_name: limit pagination_strategy: type: CustomPaginationStrategy - class_name: source_zendesk_chat.components.id_offset_pagination.IdOffsetIncrementPaginationStrategy + class_name: source_zendesk_chat.components.id_offset_pagination.ZendeskChatIdOffsetIncrementPaginationStrategy page_size: 100 - + paginator_time_offset: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: start_time + page_size_option: + inject_into: request_parameter + type: RequestOption + field_name: limit + pagination_strategy: + type: CustomPaginationStrategy + class_name: source_zendesk_chat.components.time_offset_pagination.ZendeskChatTimeOffsetIncrementPaginationStrategy + time_field_name: end_time + page_size: 1000 + # REQUESTERS requester: description: >- @@ -66,6 +81,9 @@ definitions: type: DefaultErrorHandler description: >- The default error handler + backoff_strategies: + - type: WaitTimeFromHeader + header: Retry-After # additional request kwargs request_parameters: timeout: "60" @@ -114,7 +132,20 @@ definitions: $ref: "#/definitions/retriever_for_type_list" paginator: $ref: "#/definitions/paginator_id_offset" - + base_stream_with_time_offset_pagination: + primary_key: "id" + schema_loader: + $ref: "#/definitions/schema_loader" + retriever: + $ref: "#/definitions/retriever_base" + paginator: + $ref: "#/definitions/paginator_time_offset" + requester: + $ref: "#/definitions/requester" + request_parameters: + $ref: "#/definitions/requester/request_parameters" + fields: "{{ parameters['name'] + '(*)' }}" + # INCREMENTAL base_incremental_id_stream: $ref: "#/definitions/base_stream_with_id_offset_pagination" @@ -124,10 +155,30 @@ definitions: ignore_stream_slicer_parameters_on_paginated_requests: true incremental_sync: type: CustomIncrementalSync - class_name: source_zendesk_chat.components.id_incremental.IdIncrementalCursor + class_name: source_zendesk_chat.components.id_incremental.ZendeskChatIdIncrementalCursor cursor_field: "id" field_name: "since_id" - + base_incremental_time_stream: + $ref: "#/definitions/base_stream_with_time_offset_pagination" + retriever: + $ref: "#/definitions/base_stream_with_time_offset_pagination/retriever" + # this is needed to ignore additional params for incremental syncs + ignore_stream_slicer_parameters_on_paginated_requests: true + incremental_sync: + type: CustomIncrementalSync + class_name: source_zendesk_chat.components.datetime_based_cursor.ZendeskChatTimestampCursor + use_microseconds: "{{ parameters['use_microseconds'] }}" + cursor_field: "{{ parameters['cursor_field'] }}" + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%fZ" + - "%Y-%m-%dT%H:%M:%SZ" + datetime_format: "%s" + start_datetime: + datetime: "{{ format_datetime(config['start_date'], '%s') }}" + start_time_option: + field_name: start_time + inject_into: "request_parameter" + # FULL-REFRESH STREAMS # ACCOUNTS accounts_stream: @@ -218,18 +269,47 @@ definitions: $parameters: name: "bans" path: "bans" - + # AGENTS TIMELINES + agents_timelines_stream: + description: >- + Agent Timelines Stream: https://developer.zendesk.com/rest_api/docs/chat/incremental_export#incremental-agent-timeline-export + $ref: "#/definitions/base_incremental_time_stream" + transformations: + - type: AddFields + fields: + - path: ["id"] + value: "{{ record.get('agent_id', '')|string + '|' + record.get('start_time', '')|string }}" + $parameters: + cursor_field: "start_time" + name: "agent_timeline" + data_field: "agent_timeline" + path: "incremental/agent_timeline" + use_microseconds: true + # CHATS + chats_stream: + description: >- + Chats Stream: https://developer.zendesk.com/api-reference/live-chat/chat-api/incremental_export/#incremental-chat-export + $ref: "#/definitions/base_incremental_time_stream" + $parameters: + cursor_field: "update_timestamp" + name: "chats" + data_field: "chats" + path: "incremental/chats" + use_microseconds: false + streams: - - "#/definitions/bans_stream" + - "#/definitions/accounts_stream" - "#/definitions/agents_stream" + - "#/definitions/agents_timelines_stream" + - "#/definitions/bans_stream" + - "#/definitions/chats_stream" - "#/definitions/departments_stream" - "#/definitions/goals_stream" - - "#/definitions/skills_stream" - "#/definitions/roles_stream" - - "#/definitions/triggers_stream" - - "#/definitions/shortcuts_stream" - - "#/definitions/accounts_stream" - "#/definitions/routing_settings_stream" + - "#/definitions/shortcuts_stream" + - "#/definitions/skills_stream" + - "#/definitions/triggers_stream" check: type: CheckStream diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/schemas/agent_timeline.json b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/schemas/agent_timeline.json index 4a61d458898a5..04424877eda15 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/schemas/agent_timeline.json +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/schemas/agent_timeline.json @@ -2,6 +2,9 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { + "id": { + "type": ["null", "string"] + }, "agent_id": { "type": ["null", "integer"] }, From b1ad81e26732f0e638348924082ba35500ddcfb6 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 6 Mar 2024 17:35:57 +0200 Subject: [PATCH 06/23] removed old python code base --- .../source_zendesk_chat/streams.py | 315 ------------------ 1 file changed, 315 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py deleted file mode 100644 index 353c87030e9bf..0000000000000 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py +++ /dev/null @@ -1,315 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from abc import ABC, abstractmethod -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union -from urllib.parse import parse_qs, urlparse - -import pendulum -import requests -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy -from airbyte_cdk.sources.streams.http import HttpStream - - -class Stream(HttpStream, ABC): - url_base = "https://www.zopim.com/api/v2/" - primary_key = "id" - - data_field = None - - limit = 100 - - @property - def availability_strategy(self) -> Optional["AvailabilityStrategy"]: - return None - - def request_kwargs( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Mapping[str, Any]: - - return {"timeout": 60} - - def backoff_time(self, response: requests.Response) -> Optional[float]: - delay_time = response.headers.get("Retry-After") - if delay_time: - return int(delay_time) - - def path(self, **kwargs) -> str: - return self.name - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - response_data = response.json() - - if "next_url" in response_data: - next_url = response_data["next_url"] - cursor = parse_qs(urlparse(next_url).query)["cursor"] - return {"cursor": cursor} - - def request_params( - self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = {"limit": self.limit} - if next_page_token: - params.update(next_page_token) - - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - response_data = response.json() - stream_data = self.get_stream_data(response_data) - - yield from stream_data - - def get_stream_data(self, response_data: Any) -> List[dict]: - if self.data_field: - response_data = response_data.get(self.data_field, []) - - if isinstance(response_data, list): - return list(map(self.parse_response_obj, response_data)) - elif isinstance(response_data, dict): - return [self.parse_response_obj(response_data)] - else: - raise Exception(f"Unsupported type of response data for stream {self.name}") - - def parse_response_obj(self, response_obj: dict) -> dict: - return response_obj - - -class BaseIncrementalStream(Stream, ABC): - @property - @abstractmethod - def cursor_field(self) -> str: - """ - Defining a cursor field indicates that a stream is incremental, so any incremental stream must extend this class - and define a cursor field. - """ - - @abstractmethod - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object - and returning an updated state object. - """ - - @staticmethod - def _field_to_datetime(value: Union[int, str]) -> pendulum.datetime: - if isinstance(value, int): - value = pendulum.from_timestamp(value / 1000.0) - elif isinstance(value, str): - value = pendulum.parse(value) - else: - raise ValueError(f"Unsupported type of datetime field {type(value)}") - return value - - -class TimeIncrementalStream(BaseIncrementalStream, ABC): - - state_checkpoint_interval = 1000 - - def __init__(self, start_date, **kwargs): - super().__init__(**kwargs) - self._start_date = pendulum.parse(start_date) - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - response_data = response.json() - if response_data["count"] == self.limit: - return {"start_time": response_data["end_time"]} - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - latest_benchmark = self._field_to_datetime(latest_record[self.cursor_field]) - if current_stream_state.get(self.cursor_field): - state = max(latest_benchmark, self._field_to_datetime(current_stream_state[self.cursor_field])) - return {self.cursor_field: state.strftime("%Y-%m-%dT%H:%M:%SZ")} - return {self.cursor_field: latest_benchmark.strftime("%Y-%m-%dT%H:%M:%SZ")} - - def request_params( - self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, next_page_token=next_page_token) - if next_page_token: - params.update(next_page_token) - else: - start_datetime = self._start_date - if stream_state.get(self.cursor_field): - start_datetime = pendulum.parse(stream_state[self.cursor_field]) - - params.update({"start_time": int(start_datetime.timestamp())}) - - params.update({"fields": f"{self.name}(*)"}) - return params - - def path(self, **kwargs) -> str: - return f"incremental/{self.name}" - - def parse_response_obj(self, response_obj: dict) -> dict: - response_obj[self.cursor_field] = pendulum.parse(response_obj[self.cursor_field]).strftime("%Y-%m-%dT%H:%M:%SZ") - return response_obj - - -class IdIncrementalStream(BaseIncrementalStream): - cursor_field = "id" - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - latest_benchmark = latest_record[self.cursor_field] - if current_stream_state.get(self.cursor_field): - return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])} - return {self.cursor_field: latest_benchmark} - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - stream_data = self.get_stream_data(response.json()) - if len(stream_data) == self.limit: - last_object_id = stream_data[-1]["id"] - return {"since_id": last_object_id} - - def request_params( - self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, next_page_token=next_page_token) - - if next_page_token: - params.update(next_page_token) - elif stream_state.get(self.cursor_field): - params.update({"since_id": stream_state[self.cursor_field]}) - - return params - - -class Agents(IdIncrementalStream): - """ - Agents Stream: https://developer.zendesk.com/rest_api/docs/chat/agents#list-agents - """ - - -class AgentTimelines(TimeIncrementalStream): - """ - Agent Timelines Stream: https://developer.zendesk.com/rest_api/docs/chat/incremental_export#incremental-agent-timeline-export - """ - - primary_key = None - cursor_field = "start_time" - data_field = "agent_timeline" - name = "agent_timeline" - limit = 1000 - - def request_params(self, **kwargs) -> MutableMapping[str, Any]: - params = super().request_params(**kwargs) - if not kwargs.get("next_page_token"): - params["start_time"] = params["start_time"] * 1000000 - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - response_data = response.json() - stream_data = self.get_stream_data(response_data) - - def generate_key(record): - record.update({"id": "|".join((str(record.get("agent_id", "")), str(record.get("start_time", ""))))}) - return record - - # associate the surrogate key - yield from map( - generate_key, - stream_data, - ) - - -class Accounts(Stream): - """ - Accounts Stream: https://developer.zendesk.com/rest_api/docs/chat/accounts#show-account - """ - - primary_key = "account_key" - - def path(self, **kwargs) -> str: - return "account" - - -class Chats(TimeIncrementalStream): - """ - Chats Stream: https://developer.zendesk.com/api-reference/live-chat/chat-api/incremental_export/#incremental-chat-export - """ - - cursor_field = "update_timestamp" - data_field = "chats" - limit = 1000 - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - response_data = response.json() - if response_data["count"] == self.limit: - next_page = {"start_time": response_data["end_time"]} - - start_id = response_data.get("end_id") - if start_id: - next_page.update({"start_id": start_id}) - - return next_page - - -class Shortcuts(Stream): - """ - Shortcuts Stream: https://developer.zendesk.com/rest_api/docs/chat/shortcuts#list-shortcuts - """ - - -class Triggers(Stream): - """ - Triggers Stream: https://developer.zendesk.com/rest_api/docs/chat/triggers#list-triggers - """ - - -class Bans(IdIncrementalStream): - """ - Bans Stream: https://developer.zendesk.com/rest_api/docs/chat/bans#list-bans - """ - - def get_stream_data(self, response_data) -> List[dict]: - bans = response_data["ip_address"] + response_data["visitor"] - bans = sorted(bans, key=lambda x: pendulum.parse(x["created_at"]) if x["created_at"] else pendulum.datetime(1970, 1, 1)) - return bans - - -class Departments(Stream): - """ - Departments Stream: https://developer.zendesk.com/rest_api/docs/chat/departments#list-departments - """ - - -class Goals(Stream): - """ - Goals Stream: https://developer.zendesk.com/rest_api/docs/chat/goals#list-goals - """ - - -class Skills(Stream): - """ - Skills Stream: https://developer.zendesk.com/rest_api/docs/chat/skills#list-skills - """ - - -class Roles(Stream): - """ - Roles Stream: https://developer.zendesk.com/rest_api/docs/chat/roles#list-roles - """ - - -class RoutingSettings(Stream): - """ - Routing Settings Stream: https://developer.zendesk.com/rest_api/docs/chat/routing_settings#show-account-routing-settings - """ - - primary_key = "" - - name = "routing_settings" - data_field = "data" - - def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> str: - return "routing_settings/account" From 59af274d5382b82ff3ed589a3e28b42109c4ac33 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 6 Mar 2024 18:45:47 +0200 Subject: [PATCH 07/23] format --- .../integration_tests/state.json | 26 +++++++++---------- .../components/datetime_based_cursor.py | 11 ++++---- .../components/id_incremental.py | 4 +-- .../components/id_offset_pagination.py | 8 +++--- .../components/time_offset_pagination.py | 6 ++--- .../source_zendesk_chat/manifest.yaml | 8 +++--- 6 files changed, 31 insertions(+), 32 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/state.json b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/state.json index 43873ad918475..8514c6ff16ee9 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/state.json @@ -1,14 +1,14 @@ { - "agents": { - "id": 361089721035 - }, - "bans": { - "id": 75412441 - }, - "chats": { - "update_timestamp": "2023-10-20T09:44:12Z" - }, - "agent_timeline": { - "start_time": "2024-02-09T13:12:16.817973Z" - } -} \ No newline at end of file + "agents": { + "id": 361089721035 + }, + "bans": { + "id": 75412441 + }, + "chats": { + "update_timestamp": "2023-10-20T09:44:12Z" + }, + "agent_timeline": { + "start_time": "2024-02-09T13:12:16.817973Z" + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/datetime_based_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/datetime_based_cursor.py index 7aa9ec4a160b7..1f67b9672cd62 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/datetime_based_cursor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/datetime_based_cursor.py @@ -7,8 +7,8 @@ from typing import Any, Mapping, MutableMapping, Optional, Union from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor -from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState @dataclass @@ -16,7 +16,7 @@ class ZendeskChatTimestampCursor(DatetimeBasedCursor): """ Override for the default `DatetimeBasedCursor` to make self.close_slice() to produce the STATE from the RECORD cursor value instead of slicer values. The dates in future are not allowed for the Zendesk Chat endpoints, and slicer values could be far away from exact cursor values. - + Arguments: use_microseconds: bool - whether or not to add dummy `000000` (six zeros) to provide the microseconds unit timestamps """ @@ -33,16 +33,15 @@ def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Re self._cursor = last_record_cursor_value if last_record_cursor_value else self._start_date def add_microseconds( - self, + self, field_name: str, params: MutableMapping[str, Any], - stream_slice: Optional[StreamSlice] = None, + stream_slice: Optional[StreamSlice] = None, ) -> MutableMapping[str, Any]: start_time = stream_slice.get("start_time") if start_time: params["start_time"] = int(start_time) * 1000000 return params - def get_request_params( self, @@ -55,5 +54,5 @@ def get_request_params( if self._use_microseconds: params = self.add_microseconds("start_time", params, stream_slice) else: - params['start_time'] = stream_slice.get("start_time") + params["start_time"] = stream_slice.get("start_time") return params diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental.py index f2939f0679116..d4fe3396c179e 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental.py @@ -49,14 +49,14 @@ def set_initial_state(self, stream_state: StreamState) -> None: :param stream_state: The state of the stream as returned by get_stream_state """ - + self._cursor = stream_state.get(self.cursor_field.eval(self.config)) if stream_state else None self._state = self._cursor if self._cursor else self._state def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None: last_record_cursor_value = most_recent_record.get(self.cursor_field.eval(self.config)) if most_recent_record else None self._cursor = last_record_cursor_value if last_record_cursor_value else None - + def stream_slices(self) -> Iterable[StreamSlice]: """ Use a single Slice. diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py index 0f2ead694b4ef..9c14ecbeb8f96 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py @@ -14,14 +14,14 @@ class ZendeskChatIdOffsetIncrementPaginationStrategy(OffsetIncrement): """ Id Offset Pagination docs: https://developer.zendesk.com/api-reference/live-chat/chat-api/agents/#pagination - + Attributes: page_size (InterpolatedString): the number of records to request, id_field (InterpolatedString): the name of the to track and increment from, {: 1234} """ - + id_field: str = "id" - + def __post_init__(self, parameters: Mapping[str, Any], **kwargs): self._id_field = self.id_field super().__post_init__(parameters=parameters, **kwargs) @@ -32,7 +32,7 @@ def next_page_token(self, response: requests.Response, last_records: List[Mappin if (self._page_size and len(last_records) < self._page_size.eval(self.config, response=decoded_response)) or len(last_records) == 0: return None else: - # the `IDs` are returned in `ASC` order, we add `+1` to the ID integer value to avoid the record duplicates, + # the `IDs` are returned in `ASC` order, we add `+1` to the ID integer value to avoid the record duplicates, # as described in: https://developer.zendesk.com/api-reference/live-chat/chat-api/agents/#pagination self._offset = last_records[-1][self._id_field] return self._offset + 1 diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py index cec46aaae078d..8cae6adea894f 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py @@ -14,14 +14,14 @@ class ZendeskChatTimeOffsetIncrementPaginationStrategy(OffsetIncrement): """ Time Offset Pagination docs: https://developer.zendesk.com/api-reference/live-chat/chat-api/agents/#pagination - + Attributes: page_size (InterpolatedString): the number of records to request, time_field_name (InterpolatedString): the name of the to track and increment from, {: 1234} """ - + time_field_name: str = None - + def __post_init__(self, parameters: Mapping[str, Any], **kwargs): if not self.time_field_name: raise Exception("The `time_field_name` property is missing, with no-default value.") diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml index 11895716a1b38..63c012d8f35e9 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml @@ -145,7 +145,7 @@ definitions: request_parameters: $ref: "#/definitions/requester/request_parameters" fields: "{{ parameters['name'] + '(*)' }}" - + # INCREMENTAL base_incremental_id_stream: $ref: "#/definitions/base_stream_with_id_offset_pagination" @@ -178,7 +178,7 @@ definitions: start_time_option: field_name: start_time inject_into: "request_parameter" - + # FULL-REFRESH STREAMS # ACCOUNTS accounts_stream: @@ -247,7 +247,7 @@ definitions: $parameters: name: "departments" path: "departments" - + # INCREMENTAL STREAMS # AGENTS agents_stream: @@ -314,4 +314,4 @@ streams: check: type: CheckStream stream_names: - - routing_settings \ No newline at end of file + - routing_settings From f1f6b0dc9eab55a1b71124444dfb8504bfed4c26 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 6 Mar 2024 18:55:32 +0200 Subject: [PATCH 08/23] updated references --- ...rd_extractor.py => bans_record_extractor.py} | 0 ..._incremental.py => id_incremental_cursor.py} | 0 .../source_zendesk_chat/manifest.yaml | 17 ++++++----------- 3 files changed, 6 insertions(+), 11 deletions(-) rename airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/{record_extractor.py => bans_record_extractor.py} (100%) rename airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/{id_incremental.py => id_incremental_cursor.py} (100%) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/record_extractor.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py similarity index 100% rename from airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/record_extractor.py rename to airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental_cursor.py similarity index 100% rename from airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental.py rename to airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental_cursor.py diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml index 63c012d8f35e9..0b7cc0d501936 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml @@ -16,7 +16,7 @@ definitions: type: RecordSelector extractor: type: CustomRecordExtractor - class_name: source_zendesk_chat.components.record_extractor.ZendeskChatBansRecordExtractor + class_name: source_zendesk_chat.components.bans_record_extractor.ZendeskChatBansRecordExtractor authenticator: type: BearerAuthenticator api_token: "{{ config['credentials']['access_token'] }}" @@ -119,23 +119,17 @@ definitions: retriever: $ref: "#/definitions/retriever_base" base_stream_with_list_response_no_pagination: - primary_key: "id" - schema_loader: - $ref: "#/definitions/schema_loader" + $ref: "#/definitions/base_stream" retriever: $ref: "#/definitions/retriever_for_type_list_no_pagination" base_stream_with_id_offset_pagination: - primary_key: "id" - schema_loader: - $ref: "#/definitions/schema_loader" + $ref: "#/definitions/base_stream" retriever: $ref: "#/definitions/retriever_for_type_list" paginator: $ref: "#/definitions/paginator_id_offset" base_stream_with_time_offset_pagination: - primary_key: "id" - schema_loader: - $ref: "#/definitions/schema_loader" + $ref: "#/definitions/base_stream" retriever: $ref: "#/definitions/retriever_base" paginator: @@ -144,6 +138,7 @@ definitions: $ref: "#/definitions/requester" request_parameters: $ref: "#/definitions/requester/request_parameters" + # add `fields=(*)` to the request_params fields: "{{ parameters['name'] + '(*)' }}" # INCREMENTAL @@ -155,7 +150,7 @@ definitions: ignore_stream_slicer_parameters_on_paginated_requests: true incremental_sync: type: CustomIncrementalSync - class_name: source_zendesk_chat.components.id_incremental.ZendeskChatIdIncrementalCursor + class_name: source_zendesk_chat.components.id_incremental_cursor.ZendeskChatIdIncrementalCursor cursor_field: "id" field_name: "since_id" base_incremental_time_stream: From c57c35e5a02b51089d255dd20218021052f744ca Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 7 Mar 2024 00:38:46 +0200 Subject: [PATCH 09/23] added components tests, used latest cdk version --- .../source-zendesk-chat/poetry.lock | 14 +- .../source-zendesk-chat/pyproject.toml | 2 +- .../components/id_incremental_cursor.py | 3 +- .../components/id_offset_pagination.py | 12 +- .../components/time_offset_pagination.py | 11 +- ...ed_cursor.py => timestamp_based_cursor.py} | 15 +- .../source_zendesk_chat/manifest.yaml | 17 +- .../unit_tests/__init__.py | 0 .../unit_tests/components/__init__.py | 0 .../unit_tests/components/conftest.py | 65 ++++ .../components/test_bans_record_extractor.py | 17 + .../components/test_id_incremental_cursor.py | 113 ++++++ .../components/test_id_offset_pagination.py | 32 ++ .../components/test_time_offset_pagination.py | 32 ++ .../components/test_timestamp_based_cursor.py | 67 ++++ .../unit_tests/test_source.py | 76 ---- .../unit_tests/test_streams.py | 344 ------------------ 17 files changed, 366 insertions(+), 454 deletions(-) rename airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/{datetime_based_cursor.py => timestamp_based_cursor.py} (75%) create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/__init__.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/conftest.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_bans_record_extractor.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_offset_pagination.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_time_offset_pagination.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py delete mode 100644 airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_source.py delete mode 100644 airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock b/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock index d551758b15571..8ec89910caf52 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock +++ b/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock @@ -1,14 +1,14 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "airbyte-cdk" -version = "0.65.0" +version = "0.68.2" description = "A framework for writing Airbyte Connectors." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte-cdk-0.65.0.tar.gz", hash = "sha256:068d4419227f27d9e1d9429f27c4160850a6d6208417828055b89d1f4ef3c367"}, - {file = "airbyte_cdk-0.65.0-py3-none-any.whl", hash = "sha256:8436125d39bd058ee0ffa67d47304ab5c707b3e184dd8fcd07848959aa3b0efc"}, + {file = "airbyte-cdk-0.68.2.tar.gz", hash = "sha256:04c7557e72a2b2da6ffc8abc5196f16f2c5764738284931856c9210dd2d11998"}, + {file = "airbyte_cdk-0.68.2-py3-none-any.whl", hash = "sha256:bad36c9d9a6755fe5ec2d130fa779bdf7a9248abbc8736fa4da1f35d4a97cc8e"}, ] [package.dependencies] @@ -32,8 +32,8 @@ requests-cache = "*" wcmatch = "8.4" [package.extras] -dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] -file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (>=15.0.0,<15.1.0)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (>=15.0.0,<15.1.0)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] @@ -1031,4 +1031,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9,<3.12" -content-hash = "bacda57c2899ee07da6cd98d63266c42914e1735baad9c74660ffdc7cb61cd02" +content-hash = "b98c8e95f4f109bf74d31615d43ab21a165b341629ed5a655aa0d3c7a1fd5221" diff --git a/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml b/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml index 4d2622ef7422b..2afab1fad5c4d 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml +++ b/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml @@ -17,7 +17,7 @@ include = "source_zendesk_chat" [tool.poetry.dependencies] python = "^3.9,<3.12" -airbyte-cdk = "^0.65.0" +airbyte-cdk = "^0.68.2" pendulum = "==2.1.2" [tool.poetry.scripts] diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental_cursor.py index d4fe3396c179e..28641c2eab098 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental_cursor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental_cursor.py @@ -61,7 +61,8 @@ def stream_slices(self) -> Iterable[StreamSlice]: """ Use a single Slice. """ - return [None] + slice = StreamSlice(partition={}, cursor_slice={}) + return [slice] def get_request_params( self, diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py index 9c14ecbeb8f96..8aa917114c4bd 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py @@ -3,9 +3,10 @@ # from dataclasses import dataclass -from typing import Any, List, Mapping, Optional +from typing import Any, List, Mapping, Optional, Union import requests +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.requesters.paginators.strategies import OffsetIncrement @@ -20,10 +21,13 @@ class ZendeskChatIdOffsetIncrementPaginationStrategy(OffsetIncrement): id_field (InterpolatedString): the name of the to track and increment from, {: 1234} """ - id_field: str = "id" + id_field: Union[InterpolatedString, str] = None - def __post_init__(self, parameters: Mapping[str, Any], **kwargs): - self._id_field = self.id_field + def __post_init__(self, parameters: Mapping[str, Any], **kwargs) -> None: + if not self.id_field: + raise ValueError("The `id_field` property is missing, with no-default value.") + else: + self._id_field = InterpolatedString.create(self.id_field, parameters=parameters).eval(self.config) super().__post_init__(parameters=parameters, **kwargs) def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py index 8cae6adea894f..51b5c280a7c49 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py @@ -3,9 +3,10 @@ # from dataclasses import dataclass -from typing import Any, List, Mapping, Optional +from typing import Any, List, Mapping, Optional, Union import requests +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.requesters.paginators.strategies import OffsetIncrement @@ -20,13 +21,13 @@ class ZendeskChatTimeOffsetIncrementPaginationStrategy(OffsetIncrement): time_field_name (InterpolatedString): the name of the to track and increment from, {: 1234} """ - time_field_name: str = None + time_field_name: Union[InterpolatedString, str] = None - def __post_init__(self, parameters: Mapping[str, Any], **kwargs): + def __post_init__(self, parameters: Mapping[str, Any], **kwargs) -> None: if not self.time_field_name: - raise Exception("The `time_field_name` property is missing, with no-default value.") + raise ValueError("The `time_field_name` property is missing, with no-default value.") else: - self._time_field_name = self.time_field_name + self._time_field_name = InterpolatedString.create(self.time_field_name, parameters=parameters).eval(self.config) super().__post_init__(parameters=parameters, **kwargs) def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/datetime_based_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/timestamp_based_cursor.py similarity index 75% rename from airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/datetime_based_cursor.py rename to airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/timestamp_based_cursor.py index 1f67b9672cd62..7e52f0f1c780a 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/datetime_based_cursor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/timestamp_based_cursor.py @@ -4,7 +4,7 @@ from dataclasses import dataclass -from typing import Any, Mapping, MutableMapping, Optional, Union +from typing import Any, Iterable, Mapping, MutableMapping, Optional, Union from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString @@ -29,18 +29,17 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None: super().__post_init__(parameters=parameters) def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None: - last_record_cursor_value = most_recent_record.get(self.cursor_field.eval(self.config)) if most_recent_record else None + last_record_cursor_value = most_recent_record.get(self._cursor_field.eval(self.config)) if most_recent_record else None self._cursor = last_record_cursor_value if last_record_cursor_value else self._start_date def add_microseconds( self, - field_name: str, params: MutableMapping[str, Any], stream_slice: Optional[StreamSlice] = None, ) -> MutableMapping[str, Any]: - start_time = stream_slice.get("start_time") + start_time = stream_slice.get(self._partition_field_start.eval(self.config)) if start_time: - params["start_time"] = int(start_time) * 1000000 + params[self.start_time_option.field_name.eval(config=self.config)] = int(start_time) * 1000000 return params def get_request_params( @@ -52,7 +51,9 @@ def get_request_params( ) -> Mapping[str, Any]: params = {} if self._use_microseconds: - params = self.add_microseconds("start_time", params, stream_slice) + params = self.add_microseconds(params, stream_slice) else: - params["start_time"] = stream_slice.get("start_time") + params[self.start_time_option.field_name.eval(config=self.config)] = stream_slice.get( + self._partition_field_start.eval(self.config) + ) return params diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml index 0b7cc0d501936..c715a8864a678 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml @@ -1,4 +1,4 @@ -version: 0.65.0 +version: 0.68.2 definitions: # COMMON PARTS @@ -12,11 +12,6 @@ definitions: extractor: type: DpathExtractor field_path: ["{{ parameters.get('data_field') }}"] - bans_stream_record_selector: - type: RecordSelector - extractor: - type: CustomRecordExtractor - class_name: source_zendesk_chat.components.bans_record_extractor.ZendeskChatBansRecordExtractor authenticator: type: BearerAuthenticator api_token: "{{ config['credentials']['access_token'] }}" @@ -50,6 +45,7 @@ definitions: pagination_strategy: type: CustomPaginationStrategy class_name: source_zendesk_chat.components.id_offset_pagination.ZendeskChatIdOffsetIncrementPaginationStrategy + id_field: id page_size: 100 paginator_time_offset: type: DefaultPaginator @@ -111,13 +107,13 @@ definitions: type: NoPagination # BASE STREAMS - # FULL-REFRESH base_stream: primary_key: "id" schema_loader: $ref: "#/definitions/schema_loader" retriever: $ref: "#/definitions/retriever_base" + # FULL-REFRESH base_stream_with_list_response_no_pagination: $ref: "#/definitions/base_stream" retriever: @@ -161,7 +157,7 @@ definitions: ignore_stream_slicer_parameters_on_paginated_requests: true incremental_sync: type: CustomIncrementalSync - class_name: source_zendesk_chat.components.datetime_based_cursor.ZendeskChatTimestampCursor + class_name: source_zendesk_chat.components.timestamp_based_cursor.ZendeskChatTimestampCursor use_microseconds: "{{ parameters['use_microseconds'] }}" cursor_field: "{{ parameters['cursor_field'] }}" cursor_datetime_formats: @@ -260,7 +256,10 @@ definitions: retriever: $ref: "#/definitions/base_incremental_id_stream/retriever" record_selector: - $ref: "#/definitions/bans_stream_record_selector" + type: RecordSelector + extractor: + type: CustomRecordExtractor + class_name: source_zendesk_chat.components.bans_record_extractor.ZendeskChatBansRecordExtractor $parameters: name: "bans" path: "bans" diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/__init__.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/__init__.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/conftest.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/conftest.py new file mode 100644 index 0000000000000..c48196cfa1edd --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/conftest.py @@ -0,0 +1,65 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, List, Mapping + +import pytest + + +@pytest.fixture +def config() -> Mapping[str, Any]: + return { + "start_date": "2020-10-01T00:00:00Z", + "subdomain": "", + "credentials": { + "credentials": "access_token", + "access_token": "__access_token__" + } + } + + +@pytest.fixture +def bans_stream_record() -> Mapping[str, Any]: + return { + "ip_address": [ + { + "reason": "test", + "type": "ip_address", + "id": 1234, + "created_at": "2021-04-21T14:42:46Z", + "ip_address": "0.0.0.0" + } + ], + "visitor": [ + { + "type": "visitor", + "id": 4444, + "visitor_name": "Visitor 4444", + "visitor_id": "visitor_id", + "reason": "test", + "created_at": "2021-04-27T13:25:01Z" + } + ] + } + + +@pytest.fixture +def bans_stream_record_extractor_expected_output() -> List[Mapping[str, Any]]: + return [ + { + "reason": "test", + "type": "ip_address", + "id": 1234, + "created_at": "2021-04-21T14:42:46Z", + "ip_address": "0.0.0.0" + }, + { + "type": "visitor", + "id": 4444, + "visitor_name": "Visitor 4444", + "visitor_id": "visitor_id", + "reason": "test", + "created_at": "2021-04-27T13:25:01Z" + }, + ] diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_bans_record_extractor.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_bans_record_extractor.py new file mode 100644 index 0000000000000..446bcc8f63dec --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_bans_record_extractor.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import requests +from source_zendesk_chat.components.bans_record_extractor import ZendeskChatBansRecordExtractor + + +def test_bans_stream_record_extractor( + requests_mock, + bans_stream_record, + bans_stream_record_extractor_expected_output, +) -> None: + test_url = "https://www.zopim.com/api/v2/bans" + requests_mock.get(test_url, json=bans_stream_record) + test_response = requests.get(test_url) + assert ZendeskChatBansRecordExtractor().extract_records(test_response) == bans_stream_record_extractor_expected_output diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py new file mode 100644 index 0000000000000..f8f4f703dc0f1 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py @@ -0,0 +1,113 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest +from source_zendesk_chat.components.id_incremental_cursor import ZendeskChatIdIncrementalCursor + + +def _get_cursor(config) -> ZendeskChatIdIncrementalCursor: + return ZendeskChatIdIncrementalCursor( + config = config, + cursor_field = "id", + field_name = "since_id", + parameters = {}, + ) + + +@pytest.mark.parametrize( + "stream_state, expected_cursor_value, expected_state_value", + [ + ({"id": 10}, 10, {'id': 10}), + ], + ids=[ + "SET Initial State and GET State" + ] +) +def test_id_incremental_cursor_set_initial_state_and_get_stream_state( + config, + stream_state, + expected_cursor_value, + expected_state_value, +) -> None: + cursor = _get_cursor(config) + cursor.set_initial_state(stream_state) + assert cursor._cursor == expected_cursor_value + assert cursor._state == expected_cursor_value + assert cursor.get_stream_state() == expected_state_value + + +@pytest.mark.parametrize( + "test_record, expected", + [ + ({"id": 123}, 123), + ({"id": 456}, 456), + ], + ids=[ + "first", + "second" + ] +) +def test_id_incremental_cursor_close_slice(config, test_record, expected) -> None: + cursor = _get_cursor(config) + cursor.close_slice(stream_slice={}, most_recent_record=test_record) + assert cursor._cursor == expected + + +@pytest.mark.parametrize( + "stream_state, input_slice, expected", + [ + ({}, {"id": 1}, {}), + ({"id": 2}, {"id": 1}, {"since_id": 2}), + ], + ids=[ + "No State", + "With State" + ] +) +def test_id_incremental_cursor_get_request_params(config, stream_state, input_slice, expected) -> None: + cursor = _get_cursor(config) + if stream_state: + cursor.set_initial_state(stream_state) + assert cursor.get_request_params(stream_slice=input_slice) == expected + + +@pytest.mark.parametrize( + "stream_state, record, expected", + [ + ({}, {"id": 1}, True), + ({"id": 2}, {"id": 1}, False), + ({"id": 2}, {"id": 3}, True), + ], + ids=[ + "No State", + "With State > Record value", + "With State < Record value", + ] +) +def test_id_incremental_cursor_should_be_synced(config, stream_state, record, expected) -> None: + cursor = _get_cursor(config) + if stream_state: + cursor.set_initial_state(stream_state) + assert cursor.should_be_synced(record=record) == expected + + +@pytest.mark.parametrize( + "first_record, second_record, expected", + [ + ({"id": 2}, {"id": 1}, True), + ({"id": 2}, {"id": 3}, False), + ({"id": 3}, {}, True), + ({}, {}, False), + ], + ids=[ + "First > Second - should synced", + "First < Second - should not be synced", + "Has First but no Second - should be synced", + "Has no First and has no Second - should not be synced", + ] +) +def test_id_incremental_cursor_is_greater_than_or_equal(config, first_record, second_record, expected) -> None: + cursor = _get_cursor(config) + assert cursor.is_greater_than_or_equal(first=first_record, second=second_record) == expected diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_offset_pagination.py new file mode 100644 index 0000000000000..5c5f4dd46b1ad --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_offset_pagination.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest +import requests +from source_zendesk_chat.components.id_offset_pagination import ZendeskChatIdOffsetIncrementPaginationStrategy + + +def _get_paginator(config, id_field) -> ZendeskChatIdOffsetIncrementPaginationStrategy: + return ZendeskChatIdOffsetIncrementPaginationStrategy( + config = config, + page_size = 1, + id_field = id_field, + parameters = {}, + ) + + +@pytest.mark.parametrize( + "id_field, last_records, expected", + [ + ("id", [{"id": 1}], 2), + ("id", [], None) + ], +) +def test_id_offset_increment_pagination_next_page_token(requests_mock, config, id_field, last_records, expected) -> None: + paginator = _get_paginator(config, id_field) + test_url = "https://www.zopim.com/api/v2/agents" + requests_mock.get(test_url, json=last_records) + test_response = requests.get(test_url) + assert paginator.next_page_token(test_response, last_records) == expected diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_time_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_time_offset_pagination.py new file mode 100644 index 0000000000000..086ea195fac2d --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_time_offset_pagination.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest +import requests +from source_zendesk_chat.components.time_offset_pagination import ZendeskChatTimeOffsetIncrementPaginationStrategy + + +def _get_paginator(config, time_field_name) -> ZendeskChatTimeOffsetIncrementPaginationStrategy: + return ZendeskChatTimeOffsetIncrementPaginationStrategy( + config = config, + page_size = 1, + time_field_name = time_field_name, + parameters = {}, + ) + + +@pytest.mark.parametrize( + "time_field_name, response, last_records, expected", + [ + ("end_time", {"chats":[{"update_timestamp": 1}], "end_time": 2}, [{"update_timestamp": 1}], 2), + ("end_time", {"chats":[], "end_time": 3}, [], None), + ], +) +def test_time_offset_increment_pagination_next_page_token(requests_mock, config, time_field_name, response, last_records, expected) -> None: + paginator = _get_paginator(config, time_field_name) + test_url = "https://www.zopim.com/api/v2/chats" + requests_mock.get(test_url, json=response) + test_response = requests.get(test_url) + assert paginator.next_page_token(test_response, last_records) == expected diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py new file mode 100644 index 0000000000000..7038d104718ce --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py @@ -0,0 +1,67 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest +from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType +from source_zendesk_chat.components.timestamp_based_cursor import ZendeskChatTimestampCursor + + +def _get_cursor(config, cursor_field, use_microseconds) -> ZendeskChatTimestampCursor: + cursor = ZendeskChatTimestampCursor( + start_datetime = "2020-10-01T00:00:00Z", + cursor_field = cursor_field, + datetime_format = "%s", + config = config, + parameters = {}, + use_microseconds = f"{{{ {use_microseconds} }}}", + ) + # patching missing parts + cursor.start_time_option = RequestOption( + field_name = cursor_field, + inject_into = RequestOptionType.request_parameter, + parameters={}, + ) + return cursor + + +@pytest.mark.parametrize( + "use_microseconds, input_slice, expected", + [ + (True, {"start_time": 1}, {'start_time': 1000000}), + ], +) +def test_timestamp_based_cursor_add_microseconds(config, use_microseconds, input_slice, expected) -> None: + cursor = _get_cursor(config, "start_time", use_microseconds) + test_result = cursor.add_microseconds({}, input_slice) + assert test_result == expected + + +@pytest.mark.parametrize( + "use_microseconds, input_slice, expected", + [ + (True, {"start_time": 1}, {'start_time': 1000000}), + (False, {"start_time": 1}, {'start_time': 1}), + ], + ids=[ + "WITH `use_microseconds`", + "WITHOUT `use_microseconds`", + ] +) +def test_timestamp_based_cursor_get_request_params(config, use_microseconds, input_slice, expected) -> None: + cursor = _get_cursor(config, "start_time", use_microseconds) + assert cursor.get_request_params(stream_slice=input_slice) == expected + + +@pytest.mark.parametrize( + "use_microseconds, cursor_field, test_record, expected", + [ + (True, "start_time", {"start_time": 123}, 123), + (True, "dummy_cursor", {"dummy_cursor": 456}, 456), + ], +) +def test_timestamp_based_cursor_close_slice(config, use_microseconds, cursor_field, test_record, expected) -> None: + cursor = _get_cursor(config, cursor_field, use_microseconds) + cursor.close_slice(stream_slice={}, most_recent_record=test_record) + assert cursor._cursor == expected \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_source.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_source.py deleted file mode 100644 index 4607e132314f1..0000000000000 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_source.py +++ /dev/null @@ -1,76 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from unittest.mock import patch - -import pytest -import requests -from airbyte_cdk import AirbyteLogger -from source_zendesk_chat.source import SourceZendeskChat, ZendeskAuthentication -from source_zendesk_chat.streams import ( - Accounts, - Agents, - AgentTimelines, - Bans, - Chats, - Departments, - Goals, - Roles, - RoutingSettings, - Shortcuts, - Skills, - Triggers, -) - -TEST_CONFIG: dict = { - "start_date": "2020-10-01T00:00:00Z", - "access_token": "access_token", -} -TEST_INSTANCE: SourceZendeskChat = SourceZendeskChat() - - -def test_get_auth(): - expected = {"Authorization": "Bearer access_token"} - result = ZendeskAuthentication(TEST_CONFIG).get_auth().get_auth_header() - assert expected == result - - -@pytest.mark.parametrize( - "response, check_passed", - [ - (iter({"id": 123}), True), - (requests.HTTPError(), False), - ], - ids=["Success", "Fail"], -) -def test_check(response, check_passed): - with patch.object(RoutingSettings, "read_records", return_value=response) as mock_method: - result = TEST_INSTANCE.check_connection(logger=AirbyteLogger, config=TEST_CONFIG) - mock_method.assert_called() - assert check_passed == result[0] - - -@pytest.mark.parametrize( - "stream_cls", - [ - (Accounts), - (Agents), - (AgentTimelines), - (Bans), - (Chats), - (Departments), - (Goals), - (Roles), - (RoutingSettings), - (Shortcuts), - (Skills), - (Triggers), - ], -) -def test_streams(stream_cls): - streams = TEST_INSTANCE.streams(config=TEST_CONFIG) - for stream in streams: - if stream_cls in streams: - assert isinstance(stream, stream_cls) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_streams.py deleted file mode 100644 index b90941b01c72f..0000000000000 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_streams.py +++ /dev/null @@ -1,344 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest -import requests -from source_zendesk_chat.source import ZendeskAuthentication -from source_zendesk_chat.streams import ( - Accounts, - Agents, - AgentTimelines, - Bans, - Chats, - Departments, - Goals, - Roles, - RoutingSettings, - Shortcuts, - Skills, - Triggers, -) - -TEST_CONFIG: dict = { - "start_date": "2020-10-01T00:00:00Z", - "access_token": "access_token", -} -TEST_CONFIG.update(**{"authenticator": ZendeskAuthentication(TEST_CONFIG).get_auth()}) - - -class TestFullRefreshStreams: - """ - STREAMS: - Accounts, Shortcuts, Triggers, Departments, Goals, Skills, Roles, RoutingSettings - """ - - @pytest.mark.parametrize( - "stream_cls", - [ - (Accounts), - (Departments), - (Goals), - (Roles), - (RoutingSettings), - (Shortcuts), - (Skills), - (Triggers), - ], - ) - def test_request_kwargs(self, stream_cls): - stream = stream_cls(TEST_CONFIG) - expected = {"timeout": 60} - assert expected == stream.request_kwargs(stream_state=None) - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (Accounts, "5"), - (Departments, "5"), - (Goals, "5"), - (Roles, "3"), - (RoutingSettings, "3"), - (Shortcuts, "3"), - (Skills, "1"), - (Triggers, "1"), - ], - ) - def test_backoff_time(self, requests_mock, stream_cls, expected): - stream = stream_cls(TEST_CONFIG) - url = f"{stream.url_base}{stream.path()}" - test_headers = {"Retry-After": expected} - requests_mock.get(url, headers=test_headers) - response = requests.get(url) - result = stream.backoff_time(response) - assert result == int(expected) - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (Accounts, "account"), - (Departments, "departments"), - (Goals, "goals"), - (Roles, "roles"), - (RoutingSettings, "routing_settings/account"), - (Shortcuts, "shortcuts"), - (Skills, "skills"), - (Triggers, "triggers"), - ], - ) - def test_path(self, stream_cls, expected): - stream = stream_cls(TEST_CONFIG) - result = stream.path() - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, expected_cursor", - [ - (Accounts, "MTU4MD"), - (Departments, "c1Mzc"), - (Goals, "wfHw0MzJ8"), - (Roles, "0MzJ8"), - (RoutingSettings, "MTUC4wJ8"), - (Shortcuts, "MTU4MD"), - (Skills, "c1Mzc"), - (Triggers, "0MzJ8"), - ], - ) - def test_next_page_token(self, requests_mock, stream_cls, expected_cursor): - stream = stream_cls(TEST_CONFIG) - url = f"{stream.url_base}{stream.path()}" - next_url = f"{url}/cursor.json?cursor={expected_cursor}" - test_response = {"next_url": next_url} - requests_mock.get(url, json=test_response) - response = requests.get(url) - result = stream.next_page_token(response) - assert result == {"cursor": [expected_cursor]} - - @pytest.mark.parametrize( - "stream_cls, next_page_token, expected", - [ - (Accounts, {"cursor": "MTU4MD"}, {"limit": 100, "cursor": "MTU4MD"}), - (Departments, {"cursor": "c1Mzc"}, {"limit": 100, "cursor": "c1Mzc"}), - (Goals, {"cursor": "wfHw0MzJ8"}, {"limit": 100, "cursor": "wfHw0MzJ8"}), - (Roles, {"cursor": "0MzJ8"}, {"limit": 100, "cursor": "0MzJ8"}), - (RoutingSettings, {"cursor": "MTUC4wJ8"}, {"limit": 100, "cursor": "MTUC4wJ8"}), - (Shortcuts, {"cursor": "MTU4MD"}, {"limit": 100, "cursor": "MTU4MD"}), - (Skills, {"cursor": "c1Mzc"}, {"limit": 100, "cursor": "c1Mzc"}), - (Triggers, {"cursor": "0MzJ8"}, {"limit": 100, "cursor": "0MzJ8"}), - ], - ) - def test_request_params(self, stream_cls, next_page_token, expected): - stream = stream_cls(TEST_CONFIG) - result = stream.request_params(stream_state=None, next_page_token=next_page_token) - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, test_response, expected", - [ - (Accounts, [{"id": "123"}], [{"id": "123"}]), - (Departments, {"id": "123"}, [{"id": "123"}]), - (Goals, {}, [{}]), - (Roles, [{"id": "123"}], [{"id": "123"}]), - (RoutingSettings, {"data": {"id": "123"}}, [{"id": "123"}]), - (Shortcuts, [{"id": "123"}], [{"id": "123"}]), - (Skills, [{"id": "123"}], [{"id": "123"}]), - (Triggers, [{"id": "123"}], [{"id": "123"}]), - ], - ) - def test_parse_response(self, requests_mock, stream_cls, test_response, expected): - stream = stream_cls(TEST_CONFIG) - url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json=test_response) - response = requests.get(url) - result = stream.parse_response(response) - assert list(result) == expected - - -class TestTimeIncrementalStreams: - """ - STREAMS: - AgentTimelines, Chats - """ - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (AgentTimelines, 1000), - (Chats, 1000), - ], - ) - def test_state_checkpoint_interval(self, stream_cls, expected): - stream = stream_cls(start_date=TEST_CONFIG["start_date"]) - result = stream.state_checkpoint_interval - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (AgentTimelines, "start_time"), - (Chats, "update_timestamp"), - ], - ) - def test_cursor_field(self, stream_cls, expected): - stream = stream_cls(start_date=TEST_CONFIG["start_date"]) - result = stream.cursor_field - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, test_response, expected", - [ - (AgentTimelines, {"end_time": "123"}, {"start_time": "123"}), - (Chats, {"end_time": "123"}, {"start_time": "123"}), - ], - ) - def test_next_page_token(self, requests_mock, stream_cls, test_response, expected): - stream = stream_cls(start_date=TEST_CONFIG["start_date"]) - test_response.update(**{"count": stream.limit}) - url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json=test_response) - response = requests.get(url) - result = stream.next_page_token(response) - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, current_state, last_record, expected", - [ - (AgentTimelines, {}, {"start_time": "2021-01-01"}, {"start_time": "2021-01-01T00:00:00Z"}), - (Chats, {"update_timestamp": "2022-02-02"}, {"update_timestamp": "2022-03-03"}, {"update_timestamp": "2022-03-03T00:00:00Z"}), - ], - ) - def test_get_updated_state(self, stream_cls, current_state, last_record, expected): - stream = stream_cls(start_date=TEST_CONFIG["start_date"]) - result = stream.get_updated_state(current_state, last_record) - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, stream_state, next_page_token, expected", - [ - (AgentTimelines, {}, {"start_time": "123"}, {"limit": 1000, "start_time": "123", "fields": "agent_timeline(*)"}), - (Chats, {"update_timestamp": "2022-02-02"}, {"start_time": "234"}, {"limit": 1000, "start_time": "234", "fields": "chats(*)"}), - ], - ) - def test_request_params(self, stream_cls, stream_state, next_page_token, expected): - stream = stream_cls(start_date=TEST_CONFIG["start_date"]) - result = stream.request_params(stream_state=stream_state, next_page_token=next_page_token) - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, test_response, expected", - [ - ( - AgentTimelines, - {"agent_timeline": {"id": "123", "agent_id": "test_id", "start_time": "2021-01-01"}}, - [{"id": "test_id|2021-01-01T00:00:00Z", "agent_id": "test_id", "start_time": "2021-01-01T00:00:00Z"}], - ), - ( - Chats, - {"chats": {"id": "234", "agent_id": "test_id", "update_timestamp": "2022-01-01"}}, - [{"id": "234", "agent_id": "test_id", "update_timestamp": "2022-01-01T00:00:00Z"}], - ), - ], - ) - def test_parse_response(self, requests_mock, stream_cls, test_response, expected): - stream = stream_cls(start_date=TEST_CONFIG["start_date"]) - url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json=test_response) - response = requests.get(url) - result = stream.parse_response(response) - assert list(result) == expected - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (AgentTimelines, "incremental/agent_timeline"), - (Chats, "incremental/chats"), - ], - ) - def test_path(self, stream_cls, expected): - stream = stream_cls(start_date=TEST_CONFIG["start_date"]) - result = stream.path() - assert result == expected - - -class TestIdIncrementalStreams: - """ - STREAMS: - Agents, Bans - """ - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (Agents, "agents"), - (Bans, "bans"), - ], - ) - def test_path(self, stream_cls, expected): - stream = stream_cls(TEST_CONFIG) - result = stream.path() - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (Agents, "id"), - (Bans, "id"), - ], - ) - def test_cursor_field(self, stream_cls, expected): - stream = stream_cls(TEST_CONFIG) - result = stream.cursor_field - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, current_state, last_record, expected", - [ - (Agents, {}, {"id": "1"}, {"id": "1"}), - (Bans, {"id": "1"}, {"id": "2"}, {"id": "2"}), - ], - ) - def test_get_updated_state(self, stream_cls, current_state, last_record, expected): - stream = stream_cls(TEST_CONFIG) - result = stream.get_updated_state(current_state, last_record) - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, test_response, expected", - [ - (Agents, [{"id": "2"}], {"since_id": "2"}), - ], - ) - def test_next_page_token(self, requests_mock, stream_cls, test_response, expected): - stream = stream_cls(TEST_CONFIG) - stream.limit = 1 - url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json=test_response) - response = requests.get(url) - result = stream.next_page_token(response) - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, test_response, expected", - [ - (Agents, {"id": "2"}, [{"id": "2"}]), - ], - ) - def test_parse_response(self, requests_mock, stream_cls, test_response, expected): - stream = stream_cls(TEST_CONFIG) - url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json=test_response) - response = requests.get(url) - result = stream.parse_response(response) - assert list(result) == expected - - @pytest.mark.parametrize( - "stream_cls, stream_state, next_page_token, expected", - [ - (Agents, {}, {"since_id": "1"}, {"limit": 100, "since_id": "1"}), - (Bans, {"id": "1"}, {"since_id": "2"}, {"limit": 100, "since_id": "2"}), - ], - ) - def test_request_params(self, stream_cls, stream_state, next_page_token, expected): - stream = stream_cls(TEST_CONFIG) - result = stream.request_params(stream_state=stream_state, next_page_token=next_page_token) - assert result == expected From 0bae01fc88569dc0ab5740882550df8c3cc45144 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 7 Mar 2024 00:43:16 +0200 Subject: [PATCH 10/23] added migrations.md --- docs/integrations/sources/zendesk-chat-migrations.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/integrations/sources/zendesk-chat-migrations.md diff --git a/docs/integrations/sources/zendesk-chat-migrations.md b/docs/integrations/sources/zendesk-chat-migrations.md new file mode 100644 index 0000000000000..3af506c5b4c15 --- /dev/null +++ b/docs/integrations/sources/zendesk-chat-migrations.md @@ -0,0 +1,5 @@ +# Recharge Migration Guide + +## Upgrading to 1.0.0 +This version introduces breaking changes for the STATE since the code-base changed from CDK Python to CDK Low-code. +The `Reset` for the streams is required for smooth work. From bef45ab4c83aaf957b544cd4f12f68b32355ea7e Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 7 Mar 2024 00:48:35 +0200 Subject: [PATCH 11/23] updated docs --- docs/integrations/sources/zendesk-chat-migrations.md | 2 +- docs/integrations/sources/zendesk-chat.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/integrations/sources/zendesk-chat-migrations.md b/docs/integrations/sources/zendesk-chat-migrations.md index 3af506c5b4c15..82a078c9cb016 100644 --- a/docs/integrations/sources/zendesk-chat-migrations.md +++ b/docs/integrations/sources/zendesk-chat-migrations.md @@ -1,4 +1,4 @@ -# Recharge Migration Guide +# Zendesk chat Migration Guide ## Upgrading to 1.0.0 This version introduces breaking changes for the STATE since the code-base changed from CDK Python to CDK Low-code. diff --git a/docs/integrations/sources/zendesk-chat.md b/docs/integrations/sources/zendesk-chat.md index 1baf884155190..0c1a2c45a1633 100644 --- a/docs/integrations/sources/zendesk-chat.md +++ b/docs/integrations/sources/zendesk-chat.md @@ -80,6 +80,7 @@ The connector is restricted by Zendesk's [requests limitation](https://developer | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------- | +| 1.0.0 | 2024-03-07 | [35867](https://github.com/airbytehq/airbyte/pull/35867) | Migrated to `YamlDeclarativeSource (Low-code)` Airbyte CDK | | 0.2.2 | 2024-02-12 | [35185](https://github.com/airbytehq/airbyte/pull/35185) | Manage dependencies with Poetry. | | 0.2.1 | 2023-10-20 | [31643](https://github.com/airbytehq/airbyte/pull/31643) | Upgrade base image to airbyte/python-connector-base:1.1.0 | | 0.2.0 | 2023-10-11 | [30526](https://github.com/airbytehq/airbyte/pull/30526) | Use the python connector base image, remove dockerfile and implement build_customization.py | From 811567094206b71a5ab2587cf65a2be889eb58af Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 7 Mar 2024 00:55:51 +0200 Subject: [PATCH 12/23] updated metadata.yaml --- .../connectors/source-zendesk-chat/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml index 9896737c16445..a08d17bd6b5d8 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml @@ -32,7 +32,7 @@ data: 1.0.0: message: >- This update migrates the `source-zendesk-chat` to `YamlDeclarativeSource (Low-code)`. - More info in this PR: ____ + More info in this PR: https://github.com/airbytehq/airbyte/pull/35867 upgradeDeadline: "2024-04-10" supportLevel: certified tags: From ae9c2cf83f45e249e96e42dc6b6beecbe4e5505b Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 7 Mar 2024 01:06:48 +0200 Subject: [PATCH 13/23] updated migrations.md --- docs/integrations/sources/zendesk-chat-migrations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/zendesk-chat-migrations.md b/docs/integrations/sources/zendesk-chat-migrations.md index 82a078c9cb016..1667777d4db5f 100644 --- a/docs/integrations/sources/zendesk-chat-migrations.md +++ b/docs/integrations/sources/zendesk-chat-migrations.md @@ -1,4 +1,4 @@ -# Zendesk chat Migration Guide +# Zendesk Chat Migration Guide ## Upgrading to 1.0.0 This version introduces breaking changes for the STATE since the code-base changed from CDK Python to CDK Low-code. From 4a6172087ca09f32e3b342fda868257ce8437a5f Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 7 Mar 2024 12:37:31 +0200 Subject: [PATCH 14/23] simplified the next_page_poken logic for custom components --- .../components/id_offset_pagination.py | 31 +++++++++++++++---- .../components/time_offset_pagination.py | 31 +++++++++++++++---- .../source_zendesk_chat/source.py | 2 +- 3 files changed, 51 insertions(+), 13 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py index 8aa917114c4bd..9c3eb3109f52b 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py @@ -30,13 +30,32 @@ def __post_init__(self, parameters: Mapping[str, Any], **kwargs) -> None: self._id_field = InterpolatedString.create(self.id_field, parameters=parameters).eval(self.config) super().__post_init__(parameters=parameters, **kwargs) + def should_stop_pagination(self, decoded_response: Mapping[str, Any], last_records: List[Mapping[str, Any]]) -> bool: + """ + Stop paginating when there are fewer records than the page size or the current page has no records + """ + last_records_len = len(last_records) + no_records = last_records_len == 0 + current_page_len = self._page_size.eval(self.config, response=decoded_response) + return (self._page_size and last_records_len < current_page_len) or no_records + + def get_next_page_token_offset(self, last_records: List[Mapping[str, Any]]) -> int: + """ + The `IDs` are returned in `ASC` order, we add `+1` to the ID integer value to avoid the record duplicates, + Described in: https://developer.zendesk.com/api-reference/live-chat/chat-api/agents/#pagination + + Arguments: + last_records: List[Records] -- decoded from the RESPONSE. + + Returns: + The offset value as the `next_page_token` + """ + self._offset = last_records[-1][self._id_field] + return self._offset + 1 + def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: decoded_response = self.decoder.decode(response) - # Stop paginating when there are fewer records than the page size or the current page has no records - if (self._page_size and len(last_records) < self._page_size.eval(self.config, response=decoded_response)) or len(last_records) == 0: + if self.should_stop_pagination(decoded_response, last_records): return None else: - # the `IDs` are returned in `ASC` order, we add `+1` to the ID integer value to avoid the record duplicates, - # as described in: https://developer.zendesk.com/api-reference/live-chat/chat-api/agents/#pagination - self._offset = last_records[-1][self._id_field] - return self._offset + 1 + return self.get_next_page_token_offset(last_records) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py index 51b5c280a7c49..284325c12e3b0 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py @@ -30,13 +30,32 @@ def __post_init__(self, parameters: Mapping[str, Any], **kwargs) -> None: self._time_field_name = InterpolatedString.create(self.time_field_name, parameters=parameters).eval(self.config) super().__post_init__(parameters=parameters, **kwargs) + def should_stop_pagination(self, decoded_response: Mapping[str, Any], last_records: List[Mapping[str, Any]]) -> bool: + """ + Stop paginating when there are fewer records than the page size or the current page has no records + """ + last_records_len = len(last_records) + no_records = last_records_len == 0 + current_page_len = self._page_size.eval(self.config, response=decoded_response) + return (self._page_size and last_records_len < current_page_len) or no_records + + def get_next_page_token_offset(self, decoded_response: Mapping[str, Any]) -> int: + """ + The `records` are returned in `ASC` order. + Described in: https://developer.zendesk.com/api-reference/live-chat/chat-api/incremental_export/#incremental-agent-timeline-export + + Arguments: + decoded_response: Mapping[str, Any] -- The object with RECORDS decoded from the RESPONSE. + + Returns: + The offset value as the `next_page_token` + """ + self._offset = decoded_response[self._time_field_name] + return self._offset + def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: decoded_response = self.decoder.decode(response) - # Stop paginating when there are fewer records than the page size or the current page has no records - if (self._page_size and len(last_records) < self._page_size.eval(self.config, response=decoded_response)) or len(last_records) == 0: + if self.should_stop_pagination(decoded_response, last_records): return None else: - # the `records` are returned in `ASC` order, - # as described in: https://developer.zendesk.com/api-reference/live-chat/chat-api/incremental_export/#incremental-agent-timeline-export - self._offset = decoded_response[self._time_field_name] - return self._offset + return self.get_next_page_token_offset(decoded_response) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py index 518a340254e2d..2b0540f7cd8f1 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py @@ -13,5 +13,5 @@ # Declarative Source class SourceZendeskChat(YamlDeclarativeSource): - def __init__(self): + def __init__(self) -> None: super().__init__(**{"path_to_yaml": "manifest.yaml"}) From e97030f1d04e50362c328083ea8bdbe5fa3a81a4 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 7 Mar 2024 12:41:35 +0200 Subject: [PATCH 15/23] updated metadata.yaml tag --- .../connectors/source-zendesk-chat/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml index beebc374de5b2..ae33053d2f110 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml @@ -37,5 +37,5 @@ data: supportLevel: certified tags: - language:python - - cdk:python + - cdk:low-code metadataSpecVersion: "1.0" From dd7d0efde2210eb3de0f8a0f20983b7f6b0b8940 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 13 Mar 2024 14:00:57 +0200 Subject: [PATCH 16/23] updated after the 1-st review --- .../source_zendesk_chat/components/bans_record_extractor.py | 2 +- .../source-zendesk-chat/source_zendesk_chat/manifest.yaml | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py index 688e892baaa60..2dffe978edfb7 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py @@ -18,7 +18,7 @@ class ZendeskChatBansRecordExtractor(RecordExtractor): Unnesting nested bans: `visitor`, `ip_address`. """ - def extract_records(self, response: requests.Response) -> List[Record]: + def extract_records(self, response: requests.Response) -> List[Mapping[str, Any]]: response_data = response.json() ip_address: List[Mapping[str, Any]] = response_data.get("ip_address", []) visitor: List[Mapping[str, Any]] = response_data.get("visitor", []) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml index c715a8864a678..65335210dd75f 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml @@ -80,9 +80,6 @@ definitions: backoff_strategies: - type: WaitTimeFromHeader header: Retry-After - # additional request kwargs - request_parameters: - timeout: "60" # RETRIEVERS retriever_base: @@ -133,7 +130,6 @@ definitions: requester: $ref: "#/definitions/requester" request_parameters: - $ref: "#/definitions/requester/request_parameters" # add `fields=(*)` to the request_params fields: "{{ parameters['name'] + '(*)' }}" @@ -192,7 +188,7 @@ definitions: routing_settings_stream: description: >- Routing Settings Stream: https://developer.zendesk.com/rest_api/docs/chat/routing_settings#show-account-routing-settings - $ref: "#/definitions/base_stream" + $ref: "#/definitions/base_stream_with_list_response_no_pagination" primary_key: "" $parameters: name: "routing_settings" From f43039f37830f8a47a4e8266cbe0a2668ddc6732 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 13 Mar 2024 14:21:38 +0200 Subject: [PATCH 17/23] fixed bad record selector for routing_settings --- .../source-zendesk-chat/source_zendesk_chat/manifest.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml index 65335210dd75f..35b69bdcc064e 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml @@ -189,6 +189,12 @@ definitions: description: >- Routing Settings Stream: https://developer.zendesk.com/rest_api/docs/chat/routing_settings#show-account-routing-settings $ref: "#/definitions/base_stream_with_list_response_no_pagination" + retriever: + $ref: "#/definitions/base_stream_with_list_response_no_pagination/retriever" + record_selector: + extractor: + type: DpathExtractor + field_path: ["data"] primary_key: "" $parameters: name: "routing_settings" From 8474191aaa62300ebd9976bee895a6373f94c64c Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Mon, 18 Mar 2024 16:54:04 +0200 Subject: [PATCH 18/23] updated migration guides --- .../source-zendesk-chat/metadata.yaml | 8 +++-- .../sources/zendesk-chat-migrations.md | 30 +++++++++++++++++-- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml index ae33053d2f110..da21c264a0700 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml @@ -31,9 +31,11 @@ data: breakingChanges: 1.0.0: message: >- - This update migrates the `source-zendesk-chat` to `YamlDeclarativeSource (Low-code)`. - More info in this PR: https://github.com/airbytehq/airbyte/pull/35867 - upgradeDeadline: "2024-04-10" + The source `Zendesk Chat` connector is being migrated from the Python CDK to our declarative low-code CDK. + Due to changes in State format, this migration constitutes a breaking change. + After updating, please reset your source before resuming syncs. + For more information, see our migration documentation for source `Zendesk Chat`. + upgradeDeadline: "2024-04-15" supportLevel: certified tags: - language:python diff --git a/docs/integrations/sources/zendesk-chat-migrations.md b/docs/integrations/sources/zendesk-chat-migrations.md index 1667777d4db5f..361af44f75680 100644 --- a/docs/integrations/sources/zendesk-chat-migrations.md +++ b/docs/integrations/sources/zendesk-chat-migrations.md @@ -1,5 +1,31 @@ # Zendesk Chat Migration Guide ## Upgrading to 1.0.0 -This version introduces breaking changes for the STATE since the code-base changed from CDK Python to CDK Low-code. -The `Reset` for the streams is required for smooth work. +We're continuously striving to enhance the quality and reliability of our connectors at Airbyte. As part of our commitment to delivering exceptional service, we are transitioning source `Zendesk Chat` from the Python Connector Development Kit (CDK) to our innovative low-code framework. This is part of a strategic move to streamline many processes across connectors, bolstering maintainability and freeing us to focus more of our efforts on improving the performance and features of our evolving platform and growing catalog. However, due to differences between the Python and low-code CDKs, this migration constitutes a breaking change. + +We’ve evolved and standardized how state is managed for incremental streams that are nested within a parent stream. This change impacts how individual states are tracked and stored for each partition, using a more structured approach to ensure the most granular and flexible state management. +This change will affect the [`agents`, `bans`, `agents timelines`, `chats`] streams. + +## Migration Steps + +### Refresh affected schemas and reset data + +1. Select **Connections** in the main nav bar. + 1. Select the connection(s) affected by the update. +2. Select the **Replication** tab. + 1. Select **Refresh source schema**. + 2. Select **OK**. +:::note +Any detected schema changes will be listed for your review. +::: +3. Select **Save changes** at the bottom of the page. + 1. Ensure the **Reset affected streams** option is checked. +:::note +Depending on destination type you may not be prompted to reset your data. +::: +4. Select **Save connection**. +:::note +This will reset the data in your destination and initiate a fresh sync. +::: + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). From 61c5a5c2678ab99eb42fa3166866229bebfcf9e7 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Mon, 18 Mar 2024 16:59:23 +0200 Subject: [PATCH 19/23] updated migration guides with OSS instructions --- .../sources/zendesk-chat-migrations.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/integrations/sources/zendesk-chat-migrations.md b/docs/integrations/sources/zendesk-chat-migrations.md index 361af44f75680..5a165c5c63812 100644 --- a/docs/integrations/sources/zendesk-chat-migrations.md +++ b/docs/integrations/sources/zendesk-chat-migrations.md @@ -8,6 +8,20 @@ This change will affect the [`agents`, `bans`, `agents timelines`, `chats`] stre ## Migration Steps +### For Airbyte Open Source: Update the local connector image + +Airbyte Open Source users must manually update the connector image in their local registry before proceeding with the migration. To do so: + +1. Select **Settings** in the main navbar. + 1. Select **Sources**. +2. Find `zendesk-chat` in the list of connectors. + +:::note +You will see two versions listed, the current in-use version and the latest version available. +::: + +3. Select **Change** to update your OSS version to the latest available version. + ### Refresh affected schemas and reset data 1. Select **Connections** in the main nav bar. From cfcd955e6305c35eb8726d3040a2a64b01ae1d0f Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 20 Mar 2024 16:36:02 +0200 Subject: [PATCH 20/23] updated to non-breaking changes --- .../integration_tests/state.json | 2 +- .../source-zendesk-chat/metadata.yaml | 9 ---- .../source-zendesk-chat/poetry.lock | 22 ++++----- .../source-zendesk-chat/pyproject.toml | 2 +- .../source_zendesk_chat/manifest.yaml | 7 ++- .../sources/zendesk-chat-migrations.md | 45 ------------------- 6 files changed, 19 insertions(+), 68 deletions(-) delete mode 100644 docs/integrations/sources/zendesk-chat-migrations.md diff --git a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/state.json b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/state.json index 8514c6ff16ee9..5042b1676175f 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/state.json @@ -9,6 +9,6 @@ "update_timestamp": "2023-10-20T09:44:12Z" }, "agent_timeline": { - "start_time": "2024-02-09T13:12:16.817973Z" + "start_time": "2024-02-09T13:12:16Z" } } diff --git a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml index da21c264a0700..a0624131e8303 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml @@ -27,15 +27,6 @@ data: oss: enabled: true releaseStage: generally_available - releases: - breakingChanges: - 1.0.0: - message: >- - The source `Zendesk Chat` connector is being migrated from the Python CDK to our declarative low-code CDK. - Due to changes in State format, this migration constitutes a breaking change. - After updating, please reset your source before resuming syncs. - For more information, see our migration documentation for source `Zendesk Chat`. - upgradeDeadline: "2024-04-15" supportLevel: certified tags: - language:python diff --git a/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock b/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock index 8ec89910caf52..10d2aa58d1c15 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock +++ b/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock @@ -2,13 +2,13 @@ [[package]] name = "airbyte-cdk" -version = "0.68.2" +version = "0.72.1" description = "A framework for writing Airbyte Connectors." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte-cdk-0.68.2.tar.gz", hash = "sha256:04c7557e72a2b2da6ffc8abc5196f16f2c5764738284931856c9210dd2d11998"}, - {file = "airbyte_cdk-0.68.2-py3-none-any.whl", hash = "sha256:bad36c9d9a6755fe5ec2d130fa779bdf7a9248abbc8736fa4da1f35d4a97cc8e"}, + {file = "airbyte-cdk-0.72.1.tar.gz", hash = "sha256:1dbd0a11f3784cfdd5afa9f40315c9a6123e803be91f9f861642a78e7ee14cd9"}, + {file = "airbyte_cdk-0.72.1-py3-none-any.whl", hash = "sha256:849077805442286de99f589ecba4be82491a3d9d3f516ce1a8b0cbaf303db9a4"}, ] [package.dependencies] @@ -467,13 +467,13 @@ files = [ [[package]] name = "packaging" -version = "23.2" +version = "24.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, - {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, + {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, + {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, ] [[package]] @@ -857,18 +857,18 @@ test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "tes [[package]] name = "setuptools" -version = "69.1.1" +version = "69.2.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.1.1-py3-none-any.whl", hash = "sha256:02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56"}, - {file = "setuptools-69.1.1.tar.gz", hash = "sha256:5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8"}, + {file = "setuptools-69.2.0-py3-none-any.whl", hash = "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"}, + {file = "setuptools-69.2.0.tar.gz", hash = "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -1031,4 +1031,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9,<3.12" -content-hash = "b98c8e95f4f109bf74d31615d43ab21a165b341629ed5a655aa0d3c7a1fd5221" +content-hash = "ccbf9ba9481a72f2e99d49b166340fbaca1a8ae9d6ef8990e87759d8453b287a" diff --git a/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml b/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml index 2afab1fad5c4d..133b224eedda9 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml +++ b/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml @@ -17,7 +17,7 @@ include = "source_zendesk_chat" [tool.poetry.dependencies] python = "^3.9,<3.12" -airbyte-cdk = "^0.68.2" +airbyte-cdk = "^0" pendulum = "==2.1.2" [tool.poetry.scripts] diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml index 35b69bdcc064e..5a5ff833c1e7a 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml @@ -1,4 +1,4 @@ -version: 0.68.2 +version: 0.72.1 definitions: # COMMON PARTS @@ -273,6 +273,11 @@ definitions: transformations: - type: AddFields fields: + # To preserve the non-breaking changes, the additional transformations should be applied + # 1) transform the `start_time` - cursor_field to have the old datetime format == %Y-%m-%dT%H:%M:%SZ (2023-01-01T00:00:00) + - path: ["start_time"] + value: "{{ format_datetime(record.get('start_time', config['start_date']), '%Y-%m-%dT%H:%M:%SZ') }}" + # 2) make the composite `id` field - path: ["id"] value: "{{ record.get('agent_id', '')|string + '|' + record.get('start_time', '')|string }}" $parameters: diff --git a/docs/integrations/sources/zendesk-chat-migrations.md b/docs/integrations/sources/zendesk-chat-migrations.md deleted file mode 100644 index 5a165c5c63812..0000000000000 --- a/docs/integrations/sources/zendesk-chat-migrations.md +++ /dev/null @@ -1,45 +0,0 @@ -# Zendesk Chat Migration Guide - -## Upgrading to 1.0.0 -We're continuously striving to enhance the quality and reliability of our connectors at Airbyte. As part of our commitment to delivering exceptional service, we are transitioning source `Zendesk Chat` from the Python Connector Development Kit (CDK) to our innovative low-code framework. This is part of a strategic move to streamline many processes across connectors, bolstering maintainability and freeing us to focus more of our efforts on improving the performance and features of our evolving platform and growing catalog. However, due to differences between the Python and low-code CDKs, this migration constitutes a breaking change. - -We’ve evolved and standardized how state is managed for incremental streams that are nested within a parent stream. This change impacts how individual states are tracked and stored for each partition, using a more structured approach to ensure the most granular and flexible state management. -This change will affect the [`agents`, `bans`, `agents timelines`, `chats`] streams. - -## Migration Steps - -### For Airbyte Open Source: Update the local connector image - -Airbyte Open Source users must manually update the connector image in their local registry before proceeding with the migration. To do so: - -1. Select **Settings** in the main navbar. - 1. Select **Sources**. -2. Find `zendesk-chat` in the list of connectors. - -:::note -You will see two versions listed, the current in-use version and the latest version available. -::: - -3. Select **Change** to update your OSS version to the latest available version. - -### Refresh affected schemas and reset data - -1. Select **Connections** in the main nav bar. - 1. Select the connection(s) affected by the update. -2. Select the **Replication** tab. - 1. Select **Refresh source schema**. - 2. Select **OK**. -:::note -Any detected schema changes will be listed for your review. -::: -3. Select **Save changes** at the bottom of the page. - 1. Ensure the **Reset affected streams** option is checked. -:::note -Depending on destination type you may not be prompted to reset your data. -::: -4. Select **Save connection**. -:::note -This will reset the data in your destination and initiate a fresh sync. -::: - -For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). From 7885d28f2dc20e47156196d1fec5f2e275b6019f Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 20 Mar 2024 17:17:58 +0200 Subject: [PATCH 21/23] updated --- .../integration_tests/expected_records.jsonl | 6 +++--- .../connectors/source-zendesk-chat/metadata.yaml | 2 +- docs/integrations/sources/zendesk-chat.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.jsonl index 726145d3a07fb..45c1c29ec9772 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.jsonl @@ -2,9 +2,9 @@ {"stream": "agents", "data": {"enabled": true, "create_date": "2020-11-17T23:55:24Z", "role_id": 360002848996, "first_name": "Team Airbyte", "email": "integration-test@airbyte.io", "last_name": "", "id": 360786799676, "enabled_departments": [7282618889231], "departments": [7282618889231, 5059474192015, 5060105343503, 5060005480847, 5060049125391, 5060061403535, 5060061508879, 5060049288719, 5060049443215, 5060066676751, 5060066798607, 5060071902479, 5060093166863, 5060100872591, 5060101239823, 5060072765583, 5060101350159, 5060077702799, 5060088742799, 5060103345935, 5060078913935, 5059463979663, 5060103664783, 5060079026575, 5060055796111, 5060090959759, 5059403825935, 5060108375311, 5059473809295, 5059436114575, 360003074836, 6770788212111], "skills": [], "display_name": "Team Airbyte", "last_login": "2024-02-09T13:12:16Z", "login_count": 113, "roles": {"administrator": true, "owner": false}}, "emitted_at": 1709738612909} {"stream": "agents", "data": {"enabled": true, "create_date": "2021-04-23T14:33:11Z", "role_id": 360002848976, "first_name": "Fake User number - 1", "email": "fake.user-1@email.com", "last_name": "", "id": 361084605116, "enabled_departments": [7282640316815, 7282630247567, 7282624630287], "departments": [7282640316815, 7282630247567, 7282624630287, 5060105343503, 5060005480847, 5060049125391, 5060061403535, 5060061508879, 5060049288719, 5060049443215, 5060066676751, 5060066798607, 5060071902479, 5059452990735, 5060093166863, 5060100872591, 5060101239823, 5060072765583, 5060101350159, 5060077702799, 5060088742799, 5060103345935, 5060078913935, 5060103664783, 5060079026575, 5060055796111, 5060090959759, 5059473603087, 5059403825935, 5060108375311, 5059473809295, 5059436284943, 360003074836], "skills": [1300601, 8565161], "display_name": "Fake User number - 1", "last_login": null, "login_count": 0, "roles": {"administrator": false, "owner": false}}, "emitted_at": 1709738612913} {"stream": "agents", "data": {"enabled": true, "create_date": "2021-04-23T14:34:20Z", "role_id": 360002848976, "first_name": "Fake Agent number - 1", "email": "fake.agent-1@email.com", "last_name": "", "id": 361089721035, "enabled_departments": [7282630247567], "departments": [7282630247567, 7282657193103, 5059439464079, 5060105343503, 5060005480847, 5060049125391, 5060061403535, 5060061508879, 5060049288719, 5060049443215, 5060066676751, 5060066798607, 5060071902479, 5060093166863, 5060100872591, 5060101239823, 5060072765583, 5060101350159, 5060077702799, 5060088742799, 5060103345935, 5060078913935, 5060103664783, 5060079026575, 5060055796111, 5060090959759, 5059473603087, 5060108375311, 5059473809295, 5059436114575, 5059404003599, 360003074836], "skills": [1296081, 1300641], "display_name": "Fake Agent number - 1", "last_login": null, "login_count": 0, "roles": {"administrator": false, "owner": false}}, "emitted_at": 1709738612916} -{"stream": "agent_timeline", "data": {"agent_id": 360786799676, "engagement_count": 0, "start_time": "2020-12-14T04:08:32.301292Z", "status": "invisible", "duration": 459.213926, "id": "360786799676|2020-12-14T04:08:32.301292Z"}, "emitted_at": 1709738613859} -{"stream": "agent_timeline", "data": {"agent_id": 360786799676, "engagement_count": 0, "start_time": "2020-12-14T04:17:32.387364Z", "status": "invisible", "duration": 3440.710507, "id": "360786799676|2020-12-14T04:17:32.387364Z"}, "emitted_at": 1709738613863} -{"stream": "agent_timeline", "data": {"agent_id": 360786799676, "engagement_count": 0, "start_time": "2020-12-14T18:45:37.160254Z", "status": "invisible", "duration": 520.75554, "id": "360786799676|2020-12-14T18:45:37.160254Z"}, "emitted_at": 1709738613864} +{"stream": "agent_timeline", "data": {"agent_id": 360786799676, "engagement_count": 0, "start_time": "2020-12-14T04:08:32Z", "status": "invisible", "duration": 459.213926, "id": "360786799676|2020-12-14T04:08:32Z"}, "emitted_at": 1709738613859} +{"stream": "agent_timeline", "data": {"agent_id": 360786799676, "engagement_count": 0, "start_time": "2020-12-14T04:17:32Z", "status": "invisible", "duration": 3440.710507, "id": "360786799676|2020-12-14T04:17:32Z"}, "emitted_at": 1709738613863} +{"stream": "agent_timeline", "data": {"agent_id": 360786799676, "engagement_count": 0, "start_time": "2020-12-14T18:45:37Z", "status": "invisible", "duration": 520.75554, "id": "360786799676|2020-12-14T18:45:37Z"}, "emitted_at": 1709738613864} {"stream": "bans", "data": {"created_at": "2021-04-21T14:42:46Z", "reason": "Spammer", "type": "ip_address", "id": 70519881, "ip_address": "192.123.123.5"}, "emitted_at": 1709738615366} {"stream": "bans", "data": {"created_at": "2021-04-26T13:55:20Z", "reason": "Spammer", "type": "ip_address", "id": 75112241, "ip_address": "191.121.123.5"}, "emitted_at": 1709738615369} {"stream": "bans", "data": {"created_at": "2021-04-26T13:55:30Z", "reason": "Spammer", "type": "ip_address", "id": 75112281, "ip_address": "111.121.123.5"}, "emitted_at": 1709738615371} diff --git a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml index a0624131e8303..565a509271f45 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4 - dockerImageTag: 1.0.0 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-zendesk-chat documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-chat githubIssueLabel: source-zendesk-chat diff --git a/docs/integrations/sources/zendesk-chat.md b/docs/integrations/sources/zendesk-chat.md index 0c1a2c45a1633..ef641f77cdf7a 100644 --- a/docs/integrations/sources/zendesk-chat.md +++ b/docs/integrations/sources/zendesk-chat.md @@ -80,7 +80,7 @@ The connector is restricted by Zendesk's [requests limitation](https://developer | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------- | -| 1.0.0 | 2024-03-07 | [35867](https://github.com/airbytehq/airbyte/pull/35867) | Migrated to `YamlDeclarativeSource (Low-code)` Airbyte CDK | +| 0.3.0 | 2024-03-07 | [35867](https://github.com/airbytehq/airbyte/pull/35867) | Migrated to `YamlDeclarativeSource (Low-code)` Airbyte CDK | | 0.2.2 | 2024-02-12 | [35185](https://github.com/airbytehq/airbyte/pull/35185) | Manage dependencies with Poetry. | | 0.2.1 | 2023-10-20 | [31643](https://github.com/airbytehq/airbyte/pull/31643) | Upgrade base image to airbyte/python-connector-base:1.1.0 | | 0.2.0 | 2023-10-11 | [30526](https://github.com/airbytehq/airbyte/pull/30526) | Use the python connector base image, remove dockerfile and implement build_customization.py | From 8f8663ad3c3682975c9b9ffd7e7478a75b28fa48 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 20 Mar 2024 17:24:22 +0200 Subject: [PATCH 22/23] corrected version bumps --- .../connectors/source-zendesk-chat/poetry.lock | 6 +++--- .../connectors/source-zendesk-chat/pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock b/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock index 10d2aa58d1c15..9f1120dc128c0 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock +++ b/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock @@ -2,13 +2,13 @@ [[package]] name = "airbyte-cdk" -version = "0.72.1" +version = "0.72.2" description = "A framework for writing Airbyte Connectors." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte-cdk-0.72.1.tar.gz", hash = "sha256:1dbd0a11f3784cfdd5afa9f40315c9a6123e803be91f9f861642a78e7ee14cd9"}, - {file = "airbyte_cdk-0.72.1-py3-none-any.whl", hash = "sha256:849077805442286de99f589ecba4be82491a3d9d3f516ce1a8b0cbaf303db9a4"}, + {file = "airbyte-cdk-0.72.2.tar.gz", hash = "sha256:3c06ed9c1436967ffde77b51814772dbbd79745d610bc2fe400dff9c4d7a9877"}, + {file = "airbyte_cdk-0.72.2-py3-none-any.whl", hash = "sha256:8d50773fe9ffffe9be8d6c2d2fcb10c50153833053b3ef4283fcb39c544dc4b9"}, ] [package.dependencies] diff --git a/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml b/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml index 133b224eedda9..0867e658dd228 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml +++ b/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "1.0.0" +version = "0.3.0" name = "source-zendesk-chat" description = "Source implementation for Zendesk Chat." authors = [ "Airbyte ",] From c1044dea7bebf43c1a39da3ff88200d3dd92918f Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Wed, 20 Mar 2024 22:15:24 +0200 Subject: [PATCH 23/23] fixed unit tests --- .../components/id_incremental_cursor.py | 79 ++++++++++++++----- .../components/timestamp_based_cursor.py | 12 ++- .../components/test_id_incremental_cursor.py | 3 +- .../components/test_timestamp_based_cursor.py | 13 --- 4 files changed, 68 insertions(+), 39 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental_cursor.py index 28641c2eab098..1addd15641563 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental_cursor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental_cursor.py @@ -8,7 +8,6 @@ from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type from airbyte_cdk.sources.declarative.incremental.cursor import Cursor from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString -from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState from airbyte_cdk.sources.message import MessageRepository @@ -30,17 +29,20 @@ class ZendeskChatIdIncrementalCursor(Cursor): cursor_field: Union[InterpolatedString, str] field_name: Union[InterpolatedString, str] parameters: InitVar[Mapping[str, Any]] + _highest_observed_record_cursor_value: Optional[str] = field( + repr=False, default=None + ) # tracks the latest observed datetime, which may not be safe to emit in the case of out-of-order records _cursor: Optional[str] = field(repr=False, default=None) message_repository: Optional[MessageRepository] = None def __post_init__(self, parameters: Mapping[str, Any]) -> None: self._state: Optional[int] = None - self._interpolation = JinjaInterpolation() - self.cursor_field = InterpolatedString.create(self.cursor_field, parameters=parameters) - self.field_name = InterpolatedString.create(self.field_name, parameters=parameters) + self._start_boundary: int = 0 + self.cursor_field = InterpolatedString.create(self.cursor_field, parameters=parameters).eval(self.config) + self.field_name = InterpolatedString.create(self.field_name, parameters=parameters).eval(self.config) def get_stream_state(self) -> StreamState: - return {self.cursor_field.eval(self.config): self._cursor} if self._cursor else {} + return {self.cursor_field: self._cursor} if self._cursor else {} def set_initial_state(self, stream_state: StreamState) -> None: """ @@ -50,19 +52,62 @@ def set_initial_state(self, stream_state: StreamState) -> None: :param stream_state: The state of the stream as returned by get_stream_state """ - self._cursor = stream_state.get(self.cursor_field.eval(self.config)) if stream_state else None + self._cursor = stream_state.get(self.cursor_field) if stream_state else None + self._start_boundary = self._cursor if self._cursor else 0 self._state = self._cursor if self._cursor else self._state - def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None: - last_record_cursor_value = most_recent_record.get(self.cursor_field.eval(self.config)) if most_recent_record else None - self._cursor = last_record_cursor_value if last_record_cursor_value else None + def observe(self, stream_slice: StreamSlice, record: Record) -> None: + """ + Register a record with the cursor; the cursor instance can then use it to manage the state of the in-progress stream read. + + :param record: the most recently-read record, which the cursor can use to update the stream state. Outwardly-visible changes to the + stream state may need to be deferred depending on whether the source reliably orders records by the cursor field. + """ + record_cursor_value = record.get(self.cursor_field) + if self._is_within_boundaries(record, self._start_boundary): + self._highest_observed_record_cursor_value = record_cursor_value if record_cursor_value else self._start_boundary + + def _is_within_boundaries( + self, + record: Record, + start_boundary: int, + ) -> bool: + record_cursor_value = record.get(self.cursor_field) + if not record_cursor_value: + self._send_log( + Level.WARN, + f"Could not find cursor field `{self.cursor_field}` in record. The record will not be considered when emitting sync state", + ) + return False + return start_boundary <= record_cursor_value + + def collect_cursor_values(self) -> Mapping[str, Optional[int]]: + """ + Makes the `cursor_values` using `stream_slice` and `most_recent_record`. + """ + cursor_values: dict = { + "state": self._cursor if self._cursor else self._start_boundary, + "highest_observed_record_value": self._highest_observed_record_cursor_value + if self._highest_observed_record_cursor_value + else self._start_boundary, + } + # filter out the `NONE` STATE values from the `cursor_values` + return {key: value for key, value in cursor_values.items()} + + def process_state(self, cursor_values: Optional[dict] = None) -> Optional[int]: + state_value = cursor_values.get("state") if cursor_values else 0 + highest_observed_value = cursor_values.get("highest_observed_record_value") if cursor_values else 0 + return max(state_value, highest_observed_value) + + def close_slice(self, stream_slice: StreamSlice) -> None: + cursor_values: dict = self.collect_cursor_values() + self._cursor = self.process_state(cursor_values) if cursor_values else 0 def stream_slices(self) -> Iterable[StreamSlice]: """ Use a single Slice. """ - slice = StreamSlice(partition={}, cursor_slice={}) - return [slice] + return [StreamSlice(partition={}, cursor_slice={})] def get_request_params( self, @@ -76,16 +121,15 @@ def get_request_params( def _get_request_options(self, option_type: RequestOptionType, stream_slice: StreamSlice): options = {} if self._state: - options[self.field_name.eval(self.config)] = self._state + options[self.field_name] = self._state return options def should_be_synced(self, record: Record) -> bool: - cursor_field = self.cursor_field.eval(self.config) - record_cursor_value: int = record.get(cursor_field) + record_cursor_value: int = record.get(self.cursor_field) if not record_cursor_value: self._send_log( Level.WARN, - f"Could not find cursor field `{cursor_field}` in record. The incremental sync will assume it needs to be synced", + f"Could not find cursor field `{self.cursor_field}` in record. The incremental sync will assume it needs to be synced", ) return True latest_possible_cursor_value = self._cursor if self._cursor else 0 @@ -101,9 +145,8 @@ def _send_log(self, level: Level, message: str) -> None: ) def is_greater_than_or_equal(self, first: Record, second: Record) -> bool: - cursor_field = self.cursor_field.eval(self.config) - first_cursor_value = first.get(cursor_field) - second_cursor_value = second.get(cursor_field) + first_cursor_value = first.get(self.cursor_field) + second_cursor_value = second.get(self.cursor_field) if first_cursor_value and second_cursor_value: return first_cursor_value >= second_cursor_value elif first_cursor_value: diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/timestamp_based_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/timestamp_based_cursor.py index 7e52f0f1c780a..caab6dfc3cf90 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/timestamp_based_cursor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/timestamp_based_cursor.py @@ -4,7 +4,7 @@ from dataclasses import dataclass -from typing import Any, Iterable, Mapping, MutableMapping, Optional, Union +from typing import Any, Mapping, MutableMapping, Optional, Union from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString @@ -14,8 +14,10 @@ @dataclass class ZendeskChatTimestampCursor(DatetimeBasedCursor): """ - Override for the default `DatetimeBasedCursor` to make self.close_slice() to produce the STATE from the RECORD cursor value instead of slicer values. - The dates in future are not allowed for the Zendesk Chat endpoints, and slicer values could be far away from exact cursor values. + Override for the default `DatetimeBasedCursor` to provide the `request_params["start_time"]` with added `microseconds`, as required by the API. + More info: https://developer.zendesk.com/rest_api/docs/chat/incremental_export#incremental-agent-timeline-export + + The dates in future are not(!) allowed for the Zendesk Chat endpoints, and slicer values could be far away from exact cursor values. Arguments: use_microseconds: bool - whether or not to add dummy `000000` (six zeros) to provide the microseconds unit timestamps @@ -28,10 +30,6 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None: self._start_date = self.config.get("start_date") super().__post_init__(parameters=parameters) - def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None: - last_record_cursor_value = most_recent_record.get(self._cursor_field.eval(self.config)) if most_recent_record else None - self._cursor = last_record_cursor_value if last_record_cursor_value else self._start_date - def add_microseconds( self, params: MutableMapping[str, Any], diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py index f8f4f703dc0f1..9557a312b6359 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py @@ -51,7 +51,8 @@ def test_id_incremental_cursor_set_initial_state_and_get_stream_state( ) def test_id_incremental_cursor_close_slice(config, test_record, expected) -> None: cursor = _get_cursor(config) - cursor.close_slice(stream_slice={}, most_recent_record=test_record) + cursor.observe(stream_slice={}, record=test_record) + cursor.close_slice(stream_slice={}) assert cursor._cursor == expected diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py index 7038d104718ce..a98cc8283e930 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py @@ -52,16 +52,3 @@ def test_timestamp_based_cursor_add_microseconds(config, use_microseconds, input def test_timestamp_based_cursor_get_request_params(config, use_microseconds, input_slice, expected) -> None: cursor = _get_cursor(config, "start_time", use_microseconds) assert cursor.get_request_params(stream_slice=input_slice) == expected - - -@pytest.mark.parametrize( - "use_microseconds, cursor_field, test_record, expected", - [ - (True, "start_time", {"start_time": 123}, 123), - (True, "dummy_cursor", {"dummy_cursor": 456}, 456), - ], -) -def test_timestamp_based_cursor_close_slice(config, use_microseconds, cursor_field, test_record, expected) -> None: - cursor = _get_cursor(config, cursor_field, use_microseconds) - cursor.close_slice(stream_slice={}, most_recent_record=test_record) - assert cursor._cursor == expected \ No newline at end of file