From e203e18041e674ad22b725a43d24cddea81ba215 Mon Sep 17 00:00:00 2001 From: Baz Date: Thu, 28 Mar 2024 22:08:52 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20Source=20Zendesk=20Chat:=20migra?= =?UTF-8?q?te=20to=20`YamlDeclarativeSource=20(Low-code)`=20(#35867)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../connectors/source-zendesk-chat/README.md | 2 +- .../acceptance-test-config.yml | 8 +- .../integration_tests/expected_records.jsonl | 32 ++ .../integration_tests/expected_records.txt | 34 -- .../integration_tests/state.json | 14 + .../source-zendesk-chat/metadata.yaml | 6 +- .../source-zendesk-chat/poetry.lock | 94 +++-- .../source-zendesk-chat/pyproject.toml | 4 +- .../components/bans_record_extractor.py | 27 ++ .../components/id_incremental_cursor.py | 155 ++++++++ .../components/id_offset_pagination.py | 61 ++++ .../components/time_offset_pagination.py | 61 ++++ .../components/timestamp_based_cursor.py | 57 +++ .../source_zendesk_chat/manifest.yaml | 318 ++++++++++++++++ .../schemas/agent_timeline.json | 3 + .../source_zendesk_chat/source.py | 62 +--- .../source_zendesk_chat/streams.py | 315 ---------------- .../unit_tests/__init__.py | 0 .../unit_tests/components/__init__.py | 0 .../unit_tests/components/conftest.py | 65 ++++ .../components/test_bans_record_extractor.py | 17 + .../components/test_id_incremental_cursor.py | 114 ++++++ .../components/test_id_offset_pagination.py | 32 ++ .../components/test_time_offset_pagination.py | 32 ++ .../components/test_timestamp_based_cursor.py | 54 +++ .../unit_tests/test_source.py | 76 ---- .../unit_tests/test_streams.py | 344 ------------------ docs/integrations/sources/zendesk-chat.md | 1 + 28 files changed, 1116 insertions(+), 872 deletions(-) create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.jsonl delete mode 100644 airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.txt create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/integration_tests/state.json create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental_cursor.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/timestamp_based_cursor.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml delete mode 100644 airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/__init__.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/conftest.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_bans_record_extractor.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_offset_pagination.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_time_offset_pagination.py create mode 100644 airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py delete mode 100644 airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_source.py delete mode 100644 airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-zendesk-chat/README.md b/airbyte-integrations/connectors/source-zendesk-chat/README.md index f7d40d3e06a8b..411735aa8b10c 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/README.md +++ b/airbyte-integrations/connectors/source-zendesk-chat/README.md @@ -30,7 +30,7 @@ See `sample_files/sample_config.json` for a sample config file. poetry run source-zendesk-chat spec poetry run source-zendesk-chat check --config secrets/config.json poetry run source-zendesk-chat discover --config secrets/config.json -poetry run source-zendesk-chat read --config secrets/config.json --catalog sample_files/configured_catalog.json +poetry run source-zendesk-chat read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` ### Running unit tests diff --git a/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml index 37352d28dabec..674c07534f860 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml @@ -6,8 +6,6 @@ acceptance_tests: - spec_path: "source_zendesk_chat/spec.json" connection: tests: - - config_path: "secrets/config_old.json" - status: "succeed" - config_path: "secrets/config.json" status: "succeed" - config_path: "secrets/config_oauth.json" @@ -23,12 +21,10 @@ acceptance_tests: tests: - config_path: "secrets/config.json" expect_records: - path: "integration_tests/expected_records.txt" - fail_on_extra_columns: false + path: "integration_tests/expected_records.jsonl" - config_path: "secrets/config_oauth.json" expect_records: - path: "integration_tests/expected_records.txt" - fail_on_extra_columns: false + path: "integration_tests/expected_records.jsonl" incremental: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.jsonl new file mode 100644 index 0000000000000..45c1c29ec9772 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.jsonl @@ -0,0 +1,32 @@ +{"stream": "accounts", "data": {"create_date": "2020-12-11T18:33:40Z", "account_key": "svBRNv6HoJnSZRgpf6yMmBZaFMY6s2hP", "status": "active", "plan": {"goals": 5, "long_desc": "Best for Organizations to manage Large Support Teams", "price": 70.0, "short_desc": "Ideal for Large Support Teams", "widget_customization": "full", "max_agents": 5, "sla": true, "monitoring": true, "rest_api": true, "email_reports": true, "daily_reports": true, "chat_reports": true, "agent_reports": true, "agent_leaderboard": true, "unbranding": true, "high_load": true, "ip_restriction": true, "support": true, "name": "enterprise", "max_basic_triggers": "unlimited", "max_advanced_triggers": "unlimited", "max_departments": "unlimited", "max_concurrent_chats": "unlimited", "max_history_search_days": "unlimited", "operating_hours": true, "file_upload": true, "analytics": true, "integrations": true}}, "emitted_at": 1709738612256} +{"stream": "agents", "data": {"enabled": true, "create_date": "2020-11-17T23:55:24Z", "role_id": 360002848996, "first_name": "Team Airbyte", "email": "integration-test@airbyte.io", "last_name": "", "id": 360786799676, "enabled_departments": [7282618889231], "departments": [7282618889231, 5059474192015, 5060105343503, 5060005480847, 5060049125391, 5060061403535, 5060061508879, 5060049288719, 5060049443215, 5060066676751, 5060066798607, 5060071902479, 5060093166863, 5060100872591, 5060101239823, 5060072765583, 5060101350159, 5060077702799, 5060088742799, 5060103345935, 5060078913935, 5059463979663, 5060103664783, 5060079026575, 5060055796111, 5060090959759, 5059403825935, 5060108375311, 5059473809295, 5059436114575, 360003074836, 6770788212111], "skills": [], "display_name": "Team Airbyte", "last_login": "2024-02-09T13:12:16Z", "login_count": 113, "roles": {"administrator": true, "owner": false}}, "emitted_at": 1709738612909} +{"stream": "agents", "data": {"enabled": true, "create_date": "2021-04-23T14:33:11Z", "role_id": 360002848976, "first_name": "Fake User number - 1", "email": "fake.user-1@email.com", "last_name": "", "id": 361084605116, "enabled_departments": [7282640316815, 7282630247567, 7282624630287], "departments": [7282640316815, 7282630247567, 7282624630287, 5060105343503, 5060005480847, 5060049125391, 5060061403535, 5060061508879, 5060049288719, 5060049443215, 5060066676751, 5060066798607, 5060071902479, 5059452990735, 5060093166863, 5060100872591, 5060101239823, 5060072765583, 5060101350159, 5060077702799, 5060088742799, 5060103345935, 5060078913935, 5060103664783, 5060079026575, 5060055796111, 5060090959759, 5059473603087, 5059403825935, 5060108375311, 5059473809295, 5059436284943, 360003074836], "skills": [1300601, 8565161], "display_name": "Fake User number - 1", "last_login": null, "login_count": 0, "roles": {"administrator": false, "owner": false}}, "emitted_at": 1709738612913} +{"stream": "agents", "data": {"enabled": true, "create_date": "2021-04-23T14:34:20Z", "role_id": 360002848976, "first_name": "Fake Agent number - 1", "email": "fake.agent-1@email.com", "last_name": "", "id": 361089721035, "enabled_departments": [7282630247567], "departments": [7282630247567, 7282657193103, 5059439464079, 5060105343503, 5060005480847, 5060049125391, 5060061403535, 5060061508879, 5060049288719, 5060049443215, 5060066676751, 5060066798607, 5060071902479, 5060093166863, 5060100872591, 5060101239823, 5060072765583, 5060101350159, 5060077702799, 5060088742799, 5060103345935, 5060078913935, 5060103664783, 5060079026575, 5060055796111, 5060090959759, 5059473603087, 5060108375311, 5059473809295, 5059436114575, 5059404003599, 360003074836], "skills": [1296081, 1300641], "display_name": "Fake Agent number - 1", "last_login": null, "login_count": 0, "roles": {"administrator": false, "owner": false}}, "emitted_at": 1709738612916} +{"stream": "agent_timeline", "data": {"agent_id": 360786799676, "engagement_count": 0, "start_time": "2020-12-14T04:08:32Z", "status": "invisible", "duration": 459.213926, "id": "360786799676|2020-12-14T04:08:32Z"}, "emitted_at": 1709738613859} +{"stream": "agent_timeline", "data": {"agent_id": 360786799676, "engagement_count": 0, "start_time": "2020-12-14T04:17:32Z", "status": "invisible", "duration": 3440.710507, "id": "360786799676|2020-12-14T04:17:32Z"}, "emitted_at": 1709738613863} +{"stream": "agent_timeline", "data": {"agent_id": 360786799676, "engagement_count": 0, "start_time": "2020-12-14T18:45:37Z", "status": "invisible", "duration": 520.75554, "id": "360786799676|2020-12-14T18:45:37Z"}, "emitted_at": 1709738613864} +{"stream": "bans", "data": {"created_at": "2021-04-21T14:42:46Z", "reason": "Spammer", "type": "ip_address", "id": 70519881, "ip_address": "192.123.123.5"}, "emitted_at": 1709738615366} +{"stream": "bans", "data": {"created_at": "2021-04-26T13:55:20Z", "reason": "Spammer", "type": "ip_address", "id": 75112241, "ip_address": "191.121.123.5"}, "emitted_at": 1709738615369} +{"stream": "bans", "data": {"created_at": "2021-04-26T13:55:30Z", "reason": "Spammer", "type": "ip_address", "id": 75112281, "ip_address": "111.121.123.5"}, "emitted_at": 1709738615371} +{"stream": "chats", "data": {"timestamp": "2021-04-26T13:54:02Z", "unread": false, "webpath": [], "type": "offline_msg", "id": "2104.10414779.SVhDCJ9flq79a", "update_timestamp": "2021-04-27T15:09:17Z", "tags": [], "department_name": null, "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2014-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "department_id": null, "deleted": false, "message": "Hi there!", "visitor": {"phone": "+32178763521", "notes": "Test 2", "id": "3.45678", "name": "Jiny", "email": "visitor_jiny@doe.com"}, "zendesk_ticket_id": null}, "emitted_at": 1709738618587} +{"stream": "chats", "data": {"timestamp": "2021-04-21T14:36:55Z", "unread": false, "webpath": [], "type": "offline_msg", "id": "2104.10414779.SVE9Mo9bE4wR8", "update_timestamp": "2021-04-30T11:06:19Z", "tags": [], "department_name": null, "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2014-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "department_id": null, "deleted": false, "message": "Hi there!", "visitor": {"phone": "", "notes": "", "id": "1.12345", "name": "John", "email": "visitor_john@doe.com"}, "zendesk_ticket_id": null}, "emitted_at": 1709738618592} +{"stream": "chats", "data": {"timestamp": "2021-04-26T13:53:30Z", "unread": false, "webpath": [], "type": "offline_msg", "id": "2104.10414779.SVhD3v7I1LBOq", "update_timestamp": "2021-04-30T11:08:12Z", "tags": [], "department_name": null, "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2014-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "department_id": null, "deleted": false, "message": "Hi there!", "visitor": {"phone": "+78763521", "notes": "Test", "id": "2.34567", "name": "Tiny", "email": "visitor_tiny@doe.com"}, "zendesk_ticket_id": null}, "emitted_at": 1709738618596} +{"stream": "departments", "data": {"name": "Airbyte Department 1", "members": [361084605116], "settings": {"chat_enabled": true, "solved_ticket_reassignment_strategy": "", "support_group_id": 7282640316815}, "description": "A sample department", "id": 7282640316815, "enabled": true}, "emitted_at": 1709738620228} +{"stream": "departments", "data": {"name": "Department 1", "members": [360786799676], "settings": {"chat_enabled": true, "solved_ticket_reassignment_strategy": "", "support_group_id": 7282618889231}, "description": "A sample department", "id": 7282618889231, "enabled": true}, "emitted_at": 1709738620231} +{"stream": "departments", "data": {"name": "Department 2", "members": [361089721035, 361084605116], "settings": {"chat_enabled": true, "solved_ticket_reassignment_strategy": "", "support_group_id": 7282630247567}, "description": "A sample department 2", "id": 7282630247567, "enabled": true}, "emitted_at": 1709738620233} +{"stream": "goals", "data": {"enabled": true, "attribution_model": "first_touch", "description": "A new goal", "name": "Goal 3", "id": 513481, "attribution_period": 15, "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1709738621059} +{"stream": "goals", "data": {"enabled": false, "attribution_model": "first_touch", "description": "A new goal - 1", "name": "Goal one", "id": 529641, "attribution_period": 5, "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1709738621061} +{"stream": "goals", "data": {"enabled": false, "attribution_model": "first_touch", "description": "A new goal - 2", "name": "Goal two", "id": 529681, "attribution_period": 15, "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1709738621063} +{"stream": "roles", "data": {"enabled": true, "permissions": {"visitors_seen": "account", "proactive_chatting": "listen-join", "edit_visitor_information": true, "edit_visitor_notes": true, "view_past_chats": "account", "edit_chat_tags": true, "manage_bans": "account", "access_analytics": "account", "view_monitor": "account", "edit_department_agents": "account", "set_agent_chat_limit": "account", "manage_shortcuts": "account"}, "description": "In addition to regular agent privileges, administrators can edit widget and accounts settings, manage agents, roles and permissions, and more. Permissions for the administrator role cannot be modified.", "name": "Administrator", "id": 360002848996, "members_count": 1}, "emitted_at": 1709738621711} +{"stream": "roles", "data": {"enabled": true, "permissions": {"visitors_seen": "account", "proactive_chatting": "listen-join", "edit_visitor_information": true, "edit_visitor_notes": true, "view_past_chats": "account", "edit_chat_tags": false, "manage_bans": "account", "access_analytics": "none", "view_monitor": "account", "edit_department_agents": "none", "set_agent_chat_limit": "none", "manage_shortcuts": "account"}, "description": "Agent is the most basic role in an account, and their primary responsibility is to serve chats. Permissions for the agent role can be modified.", "name": "Agent", "id": 360002848976, "members_count": 2}, "emitted_at": 1709738621714} +{"stream": "roles", "data": {"enabled": true, "permissions": {"visitors_seen": "account", "proactive_chatting": "listen-join", "edit_visitor_information": true, "edit_visitor_notes": true, "view_past_chats": "account", "edit_chat_tags": false, "manage_bans": "account", "access_analytics": "none", "view_monitor": "account", "edit_department_agents": "none", "set_agent_chat_limit": "none", "manage_shortcuts": "account"}, "description": "Can serve social messaging conversations only", "name": "Agent (limited)", "id": 7282769201935, "members_count": 0}, "emitted_at": 1709738621715} +{"stream": "routing_settings", "data": {"routing_mode": "assigned", "chat_limit": {"enabled": false, "limit": 3, "limit_type": "account", "allow_agent_override": false}, "skill_routing": {"enabled": true, "max_wait_time": 30}, "reassignment": {"enabled": true, "timeout": 30}, "auto_idle": {"enabled": false, "reassignments_before_idle": 3, "new_status": "away"}, "auto_accept": {"enabled": false}}, "emitted_at": 1709738622442} +{"stream": "shortcuts", "data": {"options": "Yes/No", "id": "goodbye", "scope": "all", "name": "goodbye", "tags": ["goodbye_survey"], "message": "Thanks for chatting with us. Have we resolved your question(s)?"}, "emitted_at": 1709738623130} +{"stream": "shortcuts", "data": {"options": "Yes/No", "id": "help", "scope": "all", "name": "help", "tags": ["help_survey"], "message": "Do you need any help?"}, "emitted_at": 1709738623133} +{"stream": "shortcuts", "data": {"options": "", "id": "hi", "scope": "all", "name": "hi", "tags": [], "message": "Hi, how can we help you today? =)"}, "emitted_at": 1709738623135} +{"stream": "skills", "data": {"id": 1300601, "name": "english", "members": [361084605116], "description": "English language", "enabled": true}, "emitted_at": 1709738623775} +{"stream": "skills", "data": {"id": 1300641, "name": "france", "members": [361089721035], "description": "France language", "enabled": true}, "emitted_at": 1709738623778} +{"stream": "skills", "data": {"id": 1296081, "name": "mandarin", "members": [361089721035], "description": "Chinese language", "enabled": true}, "emitted_at": 1709738623780} +{"stream": "triggers", "data": {"enabled": true, "definition": {"condition": ["and", ["not", ["firedBefore"]], ["and", ["neq", "@account_status", "offline"], ["stillOnSite", 60], ["eq", "@visitor_served", false]]], "event": "chat_requested", "actions": [["setTriggered", true], ["sendMessageToVisitor", "Customer Service", "We apologize for keeping you waiting. Our operators are busy at the moment, please leave us a message with your email address and we'll get back to you shortly."]]}, "description": "Auto respond to messages if agents don't respond in time.", "name": "Chat Rescuer", "id": 66052481}, "emitted_at": 1709738624471} +{"stream": "triggers", "data": {"enabled": true, "definition": {"event": "page_enter", "condition": ["and", ["eq", "@visitor_page_url", "www.zendesk.com/cart"], ["stillOnPage", 60], ["eq", "@visitor_requesting_chat", false], ["eq", "@visitor_served", false], ["not", ["firedBefore"]]], "actions": [["sendMessageToVisitor", "Stephanie", "Hi, are you having any trouble checking out? Feel free to reach out to us with any questions."]], "version": 1, "editor": "advanced"}, "description": "Reduce cart abandonment by engaging customers that are lingering on the checkout page.", "name": "Checkout Page", "id": 66052561}, "emitted_at": 1709738624474} +{"stream": "triggers", "data": {"enabled": true, "definition": {"event": "chat_requested", "condition": ["and", ["eq", "@visitor_requesting_chat", true], ["eq", "@visitor_served", false], ["not", ["firedBefore"]]], "actions": [["wait", 5], ["sendMessageToVisitor", "Customer Service", "Thanks for your message, please wait a moment while our agents attend to you."]], "version": 1, "editor": "advanced"}, "description": "Send an automated reply to customers that start a chat, so they know their request is being attended to.", "name": "First Reply", "id": 66052601}, "emitted_at": 1709738624476} diff --git a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.txt deleted file mode 100644 index 10f75ba0af99f..0000000000000 --- a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.txt +++ /dev/null @@ -1,34 +0,0 @@ -{"stream": "accounts", "data": {"create_date": "2020-12-11T18:33:40Z", "status": "active", "account_key": "svBRNv6HoJnSZRgpf6yMmBZaFMY6s2hP", "plan": {"goals": 5, "long_desc": "Best for Organizations to manage Large Support Teams", "price": 70.0, "short_desc": "Ideal for Large Support Teams", "widget_customization": "full", "max_agents": 5, "sla": true, "monitoring": true, "rest_api": true, "email_reports": true, "daily_reports": true, "chat_reports": true, "agent_reports": true, "agent_leaderboard": true, "unbranding": true, "high_load": true, "ip_restriction": true, "support": true, "name": "enterprise", "max_basic_triggers": "unlimited", "max_advanced_triggers": "unlimited", "max_departments": "unlimited", "max_concurrent_chats": "unlimited", "max_history_search_days": "unlimited", "operating_hours": true, "file_upload": true, "analytics": true, "integrations": true}}, "emitted_at": 1672828432816} -{"stream": "agent_timeline", "data": {"agent_id": 360786799676, "engagement_count": 0, "start_time": "2022-01-17T13:20:50Z", "status": "invisible", "duration": 789.733983, "id": "360786799676|2022-01-17T13:20:50Z"}, "emitted_at": 1672828433249} -{"stream": "agent_timeline", "data": {"agent_id": 360786799676, "engagement_count": 0, "start_time": "2022-06-30T17:16:55Z", "status": "invisible", "duration": 61.089883, "id": "360786799676|2022-06-30T17:16:55Z"}, "emitted_at": 1672828433249} -{"stream": "agent_timeline", "data": {"agent_id": 360786799676, "engagement_count": 0, "start_time": "2022-10-28T12:43:05Z", "status": "invisible", "duration": 370.793077, "id": "360786799676|2022-10-28T12:43:05Z"}, "emitted_at": 1672828433249} -{"stream": "agents", "data": {"role_id": 360002848976, "departments": [7282640316815, 7282630247567, 7282624630287, 5060105343503, 5060005480847, 5060049125391, 5060061403535, 5060061508879, 5060049288719, 5060049443215, 5060066676751, 5060066798607, 5060071902479, 5059452990735, 5060093166863, 5060100872591, 5060101239823, 5060072765583, 5060101350159, 5060077702799, 5060088742799, 5060103345935, 5060078913935, 5060103664783, 5060079026575, 5060055796111, 5060090959759, 5059473603087, 5059403825935, 5060108375311, 5059473809295, 5059436284943, 360003074836], "enabled_departments": [7282640316815, 7282630247567, 7282624630287], "last_name": "", "create_date": "2021-04-23T14:33:11Z", "first_name": "Fake User number - 1", "enabled": true, "skills": [1300601, 8565161], "id": 361084605116, "display_name": "Fake User number - 1", "email": "fake.user-1@email.com", "last_login": null, "login_count": 0, "roles": {"administrator": false, "owner": false}}, "emitted_at": 1688547518353} -{"stream": "agents", "data": {"role_id": 360002848976, "departments": [7282630247567, 7282657193103, 5059439464079, 5060105343503, 5060005480847, 5060049125391, 5060061403535, 5060061508879, 5060049288719, 5060049443215, 5060066676751, 5060066798607, 5060071902479, 5060093166863, 5060100872591, 5060101239823, 5060072765583, 5060101350159, 5060077702799, 5060088742799, 5060103345935, 5060078913935, 5060103664783, 5060079026575, 5060055796111, 5060090959759, 5059473603087, 5060108375311, 5059473809295, 5059436114575, 5059404003599, 360003074836], "enabled_departments": [7282630247567], "last_name": "", "create_date": "2021-04-23T14:34:20Z", "first_name": "Fake Agent number - 1", "enabled": true, "skills": [1296081, 1300641], "id": 361089721035, "display_name": "Fake Agent number - 1", "email": "fake.agent-1@email.com", "last_login": null, "login_count": 0, "roles": {"administrator": false, "owner": false}}, "emitted_at": 1688547518353} -{"stream": "bans", "data": {"type": "visitor", "id": 75411361, "reason": "Spammer", "created_at": "2021-04-27T15:52:32Z", "visitor_name": "Visitor 47225177", "visitor_id": "10414779.13ojzHu7ISdt0SM"}, "emitted_at": 1672828433831} -{"stream": "bans", "data": {"type": "visitor", "id": 75411401, "reason": "Spammer", "created_at": "2021-04-27T15:52:32Z", "visitor_name": "Visitor 62959049", "visitor_id": "10414779.13ojzHu7at4VKcG"}, "emitted_at": 1672828433831} -{"stream": "bans", "data": {"created_at": "2021-04-27T15:52:32Z", "visitor_id": "10414779.13ojzHu7at4VKcG", "id": 75411401, "reason": "Spammer", "visitor_name": "Visitor 62959049", "type": "visitor"}, "emitted_at": 1672828434000} -{"stream": "bans", "data": {"created_at": "2021-04-27T15:52:33Z", "visitor_id": "10414779.13ojzHu7s9YwIjz", "id": 75411441, "reason": "Spammer", "visitor_name": "Visitor 97350211", "type": "visitor"}, "emitted_at": 1672828434001} -{"stream": "chats", "data": {"department_id": null, "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2014-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "visitor": {"phone": "+32178763521", "notes": "Test 2", "id": "3.45678", "name": "Jiny", "email": "visitor_jiny@doe.com"}, "update_timestamp": "2021-04-27T15:09:17Z", "department_name": null, "type": "offline_msg", "deleted": false, "tags": [], "timestamp": "2021-04-26T13:54:02Z", "unread": false, "id": "2104.10414779.SVhDCJ9flq79a", "message": "Hi there!", "zendesk_ticket_id": null}, "emitted_at": 1701452730189} -{"stream": "chats", "data": {"department_id": null, "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2014-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "visitor": {"phone": "", "notes": "", "id": "1.12345", "name": "John", "email": "visitor_john@doe.com"}, "update_timestamp": "2021-04-30T11:06:19Z", "department_name": null, "type": "offline_msg", "deleted": false, "tags": [], "timestamp": "2021-04-21T14:36:55Z", "unread": false, "id": "2104.10414779.SVE9Mo9bE4wR8", "message": "Hi there!", "zendesk_ticket_id": null}, "emitted_at": 1701452730190} -{"stream": "chats", "data": {"department_id": null, "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2014-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "visitor": {"phone": "+78763521", "notes": "Test", "id": "2.34567", "name": "Tiny", "email": "visitor_tiny@doe.com"}, "update_timestamp": "2021-04-30T11:08:12Z", "department_name": null, "type": "offline_msg", "deleted": false, "tags": [], "timestamp": "2021-04-26T13:53:30Z", "unread": false, "id": "2104.10414779.SVhD3v7I1LBOq", "message": "Hi there!", "zendesk_ticket_id": null}, "emitted_at": 1701452730190} -{"stream": "chats", "data": {"department_id": null, "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2022-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "visitor": {"phone": "", "notes": "", "id": "7.34502", "name": "Fake user - chat 2", "email": "fake_user_chat_2@doe.com"}, "update_timestamp": "2021-04-30T13:32:27Z", "department_name": null, "type": "offline_msg", "deleted": false, "tags": [], "timestamp": "2021-04-30T13:32:27Z", "unread": true, "id": "2104.10414779.SW4VrjJpOq6gk", "message": "Hi there!", "zendesk_ticket_id": null}, "emitted_at": 1701452730191} -{"stream": "departments", "data": {"settings": {"chat_enabled": true, "support_group_id": 7282640316815}, "members": [361084605116], "name": "Airbyte Department 1", "enabled": true, "description": "A sample department", "id": 7282640316815}, "emitted_at": 1688547521914} -{"stream": "departments", "data": {"settings": {"chat_enabled": true, "support_group_id": 7282618889231}, "members": [360786799676], "name": "Department 1", "enabled": true, "description": "A sample department", "id": 7282618889231}, "emitted_at": 1688547521914} -{"stream": "departments", "data": {"settings": {"chat_enabled": true, "support_group_id": 7282630247567}, "members": [361089721035, 361084605116], "name": "Department 2", "enabled": true, "description": "A sample department 2", "id": 7282630247567}, "emitted_at": 1688547521914} -{"stream": "goals", "data": {"enabled": true, "id": 513481, "attribution_period": 15, "attribution_model": "first_touch", "name": "Goal 3", "description": "A new goal", "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1701453031915} -{"stream": "goals", "data": {"enabled": false, "id": 529641, "attribution_period": 5, "attribution_model": "first_touch", "name": "Goal one", "description": "A new goal - 1", "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1701453031916} -{"stream": "goals", "data": {"enabled": false, "id": 529681, "attribution_period": 15, "attribution_model": "first_touch", "name": "Goal two", "description": "A new goal - 2", "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1701453031916} -{"stream": "goals", "data": {"enabled": true, "id": 537121, "attribution_period": 30, "attribution_model": "last_touch", "name": "Test goal", "description": "Test goal", "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://zendesk.com/thanks"}]}}, "emitted_at": 1701453031916} -{"stream": "roles", "data": {"permissions": {"visitors_seen": "account", "proactive_chatting": "listen-join", "edit_visitor_information": true, "edit_visitor_notes": true, "view_past_chats": "account", "edit_chat_tags": true, "manage_bans": "account", "access_analytics": "account", "view_monitor": "account", "edit_department_agents": "account", "set_agent_chat_limit": "account", "manage_shortcuts": "account"}, "enabled": true, "description": "In addition to regular agent privileges, administrators can edit widget and accounts settings, manage agents, roles and permissions, and more. Permissions for the administrator role cannot be modified.", "id": 360002848996, "name": "Administrator", "members_count": 1}, "emitted_at": 1672828435141} -{"stream": "roles", "data": {"permissions": {"visitors_seen": "account", "proactive_chatting": "listen-join", "edit_visitor_information": true, "edit_visitor_notes": true, "view_past_chats": "account", "edit_chat_tags": false, "manage_bans": "account", "access_analytics": "none", "view_monitor": "account", "edit_department_agents": "none", "set_agent_chat_limit": "none", "manage_shortcuts": "account"}, "enabled": true, "description": "Agent is the most basic role in an account, and their primary responsibility is to serve chats. Permissions for the agent role can be modified.", "id": 360002848976, "name": "Agent", "members_count": 2}, "emitted_at": 1672828435142} -{"stream": "shortcuts", "data": {"name": "goodbye", "id": "goodbye", "options": "Yes/No", "tags": ["goodbye_survey"], "scope": "all", "message": "Thanks for chatting with us. Have we resolved your question(s)?"}, "emitted_at": 1672828435386} -{"stream": "shortcuts", "data": {"name": "help", "id": "help", "options": "Yes/No", "tags": ["help_survey"], "scope": "all", "message": "Do you need any help?"}, "emitted_at": 1672828435386} -{"stream": "shortcuts", "data": {"name": "hi", "id": "hi", "options": "", "tags": [], "scope": "all", "message": "Hi, how can we help you today? =)"}, "emitted_at": 1672828435386} -{"stream": "shortcuts", "data": {"name": "returning", "id": "returning", "options": "", "tags": ["returning_visitor"], "scope": "all", "message": "Welcome back. How can we help you today"}, "emitted_at": 1672828435387} -{"stream": "skills", "data": {"id": 1300601, "name": "english", "enabled": true, "description": "English language", "members": [361084605116]}, "emitted_at": 1672828435627} -{"stream": "skills", "data": {"id": 1300641, "name": "france", "enabled": true, "description": "France language", "members": [361089721035]}, "emitted_at": 1672828435628} -{"stream": "skills", "data": {"id": 1296081, "name": "mandarin", "enabled": true, "description": "Chinese language", "members": [361089721035]}, "emitted_at": 1672828435628} -{"stream": "triggers", "data": {"name": "Product Discounts", "enabled": true, "description": "Offer your returning customers a discount on one of your products or services. This Trigger will need to be customized based on the page.", "id": 66052801, "definition": {"event": "chat_requested", "condition": ["and", ["icontains", "@visitor_page_url", "[product name]"], ["stillOnPage", 30], ["eq", "@visitor_requesting_chat", false], ["eq", "@visitor_served", false], ["not", ["firedBefore"]]], "actions": [["sendMessageToVisitor", "Customer Service", "Hi, are you interested in [insert product name]? We're offering a one-time 20% discount. Chat with me to find out more."]], "version": 1, "editor": "advanced"}}, "emitted_at": 1688547525543} -{"stream": "triggers", "data": {"name": "Request Contact Details", "enabled": true, "description": "When your account is set to away, ask customer's requesting a chat to leave their email address.", "id": 66052841, "definition": {"event": "chat_requested", "condition": ["and", ["eq", "@account_status", "away"], ["not", ["firedBefore"]]], "actions": [["addTag", "Away_request"], ["sendMessageToVisitor", "Customer Service", "Hi, sorry we are away at the moment. Please leave your email address and we will get back to you as soon as possible."]], "version": 1, "editor": "advanced"}}, "emitted_at": 1688547525543} -{"stream": "triggers", "data": {"name": "Tag Repeat Visitors", "enabled": true, "description": "Add a tag to a visitor that has visited your site 5 or more times. This helps you identify potential customers who are very interested in your brand.", "id": 66052881, "definition": {"event": "page_enter", "condition": ["and", ["gte", "@visitor_previous_visits", 5]], "actions": [["addTag", "5times"]], "version": 1, "editor": "advanced"}}, "emitted_at": 1688547525543} -{"stream": "routing_settings", "data": {"routing_mode": "assigned", "chat_limit": {"enabled": false, "limit": 3, "limit_type": "account", "allow_agent_override": false}, "skill_routing": {"enabled": true, "max_wait_time": 30}, "reassignment": {"enabled": true, "timeout": 30}, "auto_idle": {"enabled": false, "reassignments_before_idle": 3, "new_status": "away"}, "auto_accept": {"enabled": false}}, "emitted_at": 1701453336379} diff --git a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/state.json b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/state.json new file mode 100644 index 0000000000000..5042b1676175f --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/state.json @@ -0,0 +1,14 @@ +{ + "agents": { + "id": 361089721035 + }, + "bans": { + "id": 75412441 + }, + "chats": { + "update_timestamp": "2023-10-20T09:44:12Z" + }, + "agent_timeline": { + "start_time": "2024-02-09T13:12:16Z" + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml index 3657e930c7c41..565a509271f45 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml @@ -6,11 +6,11 @@ data: hosts: - zopim.com connectorBuildOptions: - baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4 - dockerImageTag: 0.2.2 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-zendesk-chat documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-chat githubIssueLabel: source-zendesk-chat @@ -30,5 +30,5 @@ data: supportLevel: certified tags: - language:python - - cdk:python + - cdk:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock b/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock index 4035ad70602bc..9f1120dc128c0 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock +++ b/airbyte-integrations/connectors/source-zendesk-chat/poetry.lock @@ -1,18 +1,18 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "airbyte-cdk" -version = "0.51.41" +version = "0.72.2" description = "A framework for writing Airbyte Connectors." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte-cdk-0.51.41.tar.gz", hash = "sha256:cce614d67872cf66a151e5b72d70f4bf26e2a1ce672c7abfc15a5cb4e45d8429"}, - {file = "airbyte_cdk-0.51.41-py3-none-any.whl", hash = "sha256:bbf82a45d9ec97c4a92b85e3312b327f8060fffec1f7c7ea7dfa720f9adcc13b"}, + {file = "airbyte-cdk-0.72.2.tar.gz", hash = "sha256:3c06ed9c1436967ffde77b51814772dbbd79745d610bc2fe400dff9c4d7a9877"}, + {file = "airbyte_cdk-0.72.2-py3-none-any.whl", hash = "sha256:8d50773fe9ffffe9be8d6c2d2fcb10c50153833053b3ef4283fcb39c544dc4b9"}, ] [package.dependencies] -airbyte-protocol-models = "0.4.2" +airbyte-protocol-models = "0.5.1" backoff = "*" cachetools = "*" Deprecated = ">=1.2,<2.0" @@ -22,8 +22,9 @@ isodate = ">=0.6.1,<0.7.0" Jinja2 = ">=3.1.2,<3.2.0" jsonref = ">=0.2,<1.0" jsonschema = ">=3.2.0,<3.3.0" -pendulum = "*" +pendulum = "<3.0.0" pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" python-dateutil = "*" PyYAML = ">=6.0.1" requests = "*" @@ -31,20 +32,20 @@ requests-cache = "*" wcmatch = "8.4" [package.extras] -dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pyarrow (==12.0.1)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)"] -file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "pyarrow (==12.0.1)"] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (>=15.0.0,<15.1.0)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (>=15.0.0,<15.1.0)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] [[package]] name = "airbyte-protocol-models" -version = "0.4.2" +version = "0.5.1" description = "Declares the Airbyte Protocol." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte_protocol_models-0.4.2-py3-none-any.whl", hash = "sha256:d3bbb14d4af9483bd7b08f5eb06f87e7113553bf4baed3998af95be873a0d821"}, - {file = "airbyte_protocol_models-0.4.2.tar.gz", hash = "sha256:67b149d4812f8fdb88396b161274aa73cf0e16f22e35ce44f2bfc4d47e51915c"}, + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, ] [package.dependencies] @@ -103,13 +104,13 @@ files = [ [[package]] name = "cachetools" -version = "5.3.2" +version = "5.3.3" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" files = [ - {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, - {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, + {file = "cachetools-5.3.3-py3-none-any.whl", hash = "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945"}, + {file = "cachetools-5.3.3.tar.gz", hash = "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105"}, ] [[package]] @@ -466,13 +467,13 @@ files = [ [[package]] name = "packaging" -version = "23.2" +version = "24.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, - {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, + {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, + {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, ] [[package]] @@ -602,6 +603,21 @@ typing-extensions = ">=4.2.0" dotenv = ["python-dotenv (>=0.10.4)"] email = ["email-validator (>=1.0.3)"] +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + [[package]] name = "pyrsistent" version = "0.20.0" @@ -686,13 +702,13 @@ dev = ["pre-commit", "pytest-asyncio", "tox"] [[package]] name = "python-dateutil" -version = "2.8.2" +version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, ] [package.dependencies] @@ -792,13 +808,13 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requests-cache" -version = "1.1.1" +version = "1.2.0" description = "A persistent cache for python requests" optional = false -python-versions = ">=3.7,<4.0" +python-versions = ">=3.8" files = [ - {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, - {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, + {file = "requests_cache-1.2.0-py3-none-any.whl", hash = "sha256:490324301bf0cb924ff4e6324bd2613453e7e1f847353928b08adb0fdfb7f722"}, + {file = "requests_cache-1.2.0.tar.gz", hash = "sha256:db1c709ca343cc1cd5b6c8b1a5387298eceed02306a6040760db538c885e3838"}, ] [package.dependencies] @@ -810,15 +826,15 @@ url-normalize = ">=1.4" urllib3 = ">=1.25.5" [package.extras] -all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=6.0.1)", "redis (>=3)", "ujson (>=5.4)"] bson = ["bson (>=0.5)"] -docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.9)"] dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] json = ["ujson (>=5.4)"] mongodb = ["pymongo (>=3)"] redis = ["redis (>=3)"] security = ["itsdangerous (>=2.0)"] -yaml = ["pyyaml (>=5.4)"] +yaml = ["pyyaml (>=6.0.1)"] [[package]] name = "requests-mock" @@ -841,19 +857,19 @@ test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "tes [[package]] name = "setuptools" -version = "69.1.0" +version = "69.2.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, - {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, + {file = "setuptools-69.2.0-py3-none-any.whl", hash = "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"}, + {file = "setuptools-69.2.0.tar.gz", hash = "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" @@ -879,13 +895,13 @@ files = [ [[package]] name = "typing-extensions" -version = "4.9.0" +version = "4.10.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, - {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, + {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, + {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, ] [[package]] @@ -904,13 +920,13 @@ six = "*" [[package]] name = "urllib3" -version = "2.2.0" +version = "2.2.1" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.8" files = [ - {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, - {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, ] [package.extras] @@ -1015,4 +1031,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9,<3.12" -content-hash = "e55b65b435ed00315a8288393c1fb2adde5904ae32b5aed66f133bdb721a6991" +content-hash = "ccbf9ba9481a72f2e99d49b166340fbaca1a8ae9d6ef8990e87759d8453b287a" diff --git a/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml b/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml index f47dbc02c81db..0867e658dd228 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml +++ b/airbyte-integrations/connectors/source-zendesk-chat/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "0.2.2" +version = "0.3.0" name = "source-zendesk-chat" description = "Source implementation for Zendesk Chat." authors = [ "Airbyte ",] @@ -17,7 +17,7 @@ include = "source_zendesk_chat" [tool.poetry.dependencies] python = "^3.9,<3.12" -airbyte-cdk = "==0.51.41" +airbyte-cdk = "^0" pendulum = "==2.1.2" [tool.poetry.scripts] diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py new file mode 100644 index 0000000000000..2dffe978edfb7 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/bans_record_extractor.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from dataclasses import dataclass +from typing import Any, List, Mapping + +import pendulum +import requests +from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor +from airbyte_cdk.sources.declarative.types import Record + + +@dataclass +class ZendeskChatBansRecordExtractor(RecordExtractor): + """ + Unnesting nested bans: `visitor`, `ip_address`. + """ + + def extract_records(self, response: requests.Response) -> List[Mapping[str, Any]]: + response_data = response.json() + ip_address: List[Mapping[str, Any]] = response_data.get("ip_address", []) + visitor: List[Mapping[str, Any]] = response_data.get("visitor", []) + bans = ip_address + visitor + bans = sorted(bans, key=lambda x: pendulum.parse(x["created_at"]) if x["created_at"] else pendulum.datetime(1970, 1, 1)) + return bans diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental_cursor.py new file mode 100644 index 0000000000000..1addd15641563 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_incremental_cursor.py @@ -0,0 +1,155 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import InitVar, dataclass, field +from typing import Any, Iterable, Mapping, Optional, Union + +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type +from airbyte_cdk.sources.declarative.incremental.cursor import Cursor +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType +from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState +from airbyte_cdk.sources.message import MessageRepository + + +@dataclass +class ZendeskChatIdIncrementalCursor(Cursor): + """ + Custom Incremental Cursor implementation to provide the ability to pull data using `id`(int) as cursor. + More info: https://developer.zendesk.com/api-reference/live-chat/chat-api/agents/#parameters + + Attributes: + config (Config): connection config + field_name (Union[InterpolatedString, str]): the name of the field which will hold the cursor value for outbound API call + cursor_field (Union[InterpolatedString, str]): record's cursor field + """ + + config: Config + cursor_field: Union[InterpolatedString, str] + field_name: Union[InterpolatedString, str] + parameters: InitVar[Mapping[str, Any]] + _highest_observed_record_cursor_value: Optional[str] = field( + repr=False, default=None + ) # tracks the latest observed datetime, which may not be safe to emit in the case of out-of-order records + _cursor: Optional[str] = field(repr=False, default=None) + message_repository: Optional[MessageRepository] = None + + def __post_init__(self, parameters: Mapping[str, Any]) -> None: + self._state: Optional[int] = None + self._start_boundary: int = 0 + self.cursor_field = InterpolatedString.create(self.cursor_field, parameters=parameters).eval(self.config) + self.field_name = InterpolatedString.create(self.field_name, parameters=parameters).eval(self.config) + + def get_stream_state(self) -> StreamState: + return {self.cursor_field: self._cursor} if self._cursor else {} + + def set_initial_state(self, stream_state: StreamState) -> None: + """ + Cursors are not initialized with their state. As state is needed in order to function properly, this method should be called + before calling anything else + + :param stream_state: The state of the stream as returned by get_stream_state + """ + + self._cursor = stream_state.get(self.cursor_field) if stream_state else None + self._start_boundary = self._cursor if self._cursor else 0 + self._state = self._cursor if self._cursor else self._state + + def observe(self, stream_slice: StreamSlice, record: Record) -> None: + """ + Register a record with the cursor; the cursor instance can then use it to manage the state of the in-progress stream read. + + :param record: the most recently-read record, which the cursor can use to update the stream state. Outwardly-visible changes to the + stream state may need to be deferred depending on whether the source reliably orders records by the cursor field. + """ + record_cursor_value = record.get(self.cursor_field) + if self._is_within_boundaries(record, self._start_boundary): + self._highest_observed_record_cursor_value = record_cursor_value if record_cursor_value else self._start_boundary + + def _is_within_boundaries( + self, + record: Record, + start_boundary: int, + ) -> bool: + record_cursor_value = record.get(self.cursor_field) + if not record_cursor_value: + self._send_log( + Level.WARN, + f"Could not find cursor field `{self.cursor_field}` in record. The record will not be considered when emitting sync state", + ) + return False + return start_boundary <= record_cursor_value + + def collect_cursor_values(self) -> Mapping[str, Optional[int]]: + """ + Makes the `cursor_values` using `stream_slice` and `most_recent_record`. + """ + cursor_values: dict = { + "state": self._cursor if self._cursor else self._start_boundary, + "highest_observed_record_value": self._highest_observed_record_cursor_value + if self._highest_observed_record_cursor_value + else self._start_boundary, + } + # filter out the `NONE` STATE values from the `cursor_values` + return {key: value for key, value in cursor_values.items()} + + def process_state(self, cursor_values: Optional[dict] = None) -> Optional[int]: + state_value = cursor_values.get("state") if cursor_values else 0 + highest_observed_value = cursor_values.get("highest_observed_record_value") if cursor_values else 0 + return max(state_value, highest_observed_value) + + def close_slice(self, stream_slice: StreamSlice) -> None: + cursor_values: dict = self.collect_cursor_values() + self._cursor = self.process_state(cursor_values) if cursor_values else 0 + + def stream_slices(self) -> Iterable[StreamSlice]: + """ + Use a single Slice. + """ + return [StreamSlice(partition={}, cursor_slice={})] + + def get_request_params( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + return self._get_request_options(RequestOptionType.request_parameter, stream_slice) + + def _get_request_options(self, option_type: RequestOptionType, stream_slice: StreamSlice): + options = {} + if self._state: + options[self.field_name] = self._state + return options + + def should_be_synced(self, record: Record) -> bool: + record_cursor_value: int = record.get(self.cursor_field) + if not record_cursor_value: + self._send_log( + Level.WARN, + f"Could not find cursor field `{self.cursor_field}` in record. The incremental sync will assume it needs to be synced", + ) + return True + latest_possible_cursor_value = self._cursor if self._cursor else 0 + return latest_possible_cursor_value <= record_cursor_value + + def _send_log(self, level: Level, message: str) -> None: + if self.message_repository: + self.message_repository.emit_message( + AirbyteMessage( + type=Type.LOG, + log=AirbyteLogMessage(level=level, message=message), + ) + ) + + def is_greater_than_or_equal(self, first: Record, second: Record) -> bool: + first_cursor_value = first.get(self.cursor_field) + second_cursor_value = second.get(self.cursor_field) + if first_cursor_value and second_cursor_value: + return first_cursor_value >= second_cursor_value + elif first_cursor_value: + return True + else: + return False diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py new file mode 100644 index 0000000000000..9c3eb3109f52b --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/id_offset_pagination.py @@ -0,0 +1,61 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import dataclass +from typing import Any, List, Mapping, Optional, Union + +import requests +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.requesters.paginators.strategies import OffsetIncrement + + +@dataclass +class ZendeskChatIdOffsetIncrementPaginationStrategy(OffsetIncrement): + """ + Id Offset Pagination docs: + https://developer.zendesk.com/api-reference/live-chat/chat-api/agents/#pagination + + Attributes: + page_size (InterpolatedString): the number of records to request, + id_field (InterpolatedString): the name of the to track and increment from, {: 1234} + """ + + id_field: Union[InterpolatedString, str] = None + + def __post_init__(self, parameters: Mapping[str, Any], **kwargs) -> None: + if not self.id_field: + raise ValueError("The `id_field` property is missing, with no-default value.") + else: + self._id_field = InterpolatedString.create(self.id_field, parameters=parameters).eval(self.config) + super().__post_init__(parameters=parameters, **kwargs) + + def should_stop_pagination(self, decoded_response: Mapping[str, Any], last_records: List[Mapping[str, Any]]) -> bool: + """ + Stop paginating when there are fewer records than the page size or the current page has no records + """ + last_records_len = len(last_records) + no_records = last_records_len == 0 + current_page_len = self._page_size.eval(self.config, response=decoded_response) + return (self._page_size and last_records_len < current_page_len) or no_records + + def get_next_page_token_offset(self, last_records: List[Mapping[str, Any]]) -> int: + """ + The `IDs` are returned in `ASC` order, we add `+1` to the ID integer value to avoid the record duplicates, + Described in: https://developer.zendesk.com/api-reference/live-chat/chat-api/agents/#pagination + + Arguments: + last_records: List[Records] -- decoded from the RESPONSE. + + Returns: + The offset value as the `next_page_token` + """ + self._offset = last_records[-1][self._id_field] + return self._offset + 1 + + def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: + decoded_response = self.decoder.decode(response) + if self.should_stop_pagination(decoded_response, last_records): + return None + else: + return self.get_next_page_token_offset(last_records) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py new file mode 100644 index 0000000000000..284325c12e3b0 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/time_offset_pagination.py @@ -0,0 +1,61 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import dataclass +from typing import Any, List, Mapping, Optional, Union + +import requests +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.requesters.paginators.strategies import OffsetIncrement + + +@dataclass +class ZendeskChatTimeOffsetIncrementPaginationStrategy(OffsetIncrement): + """ + Time Offset Pagination docs: + https://developer.zendesk.com/api-reference/live-chat/chat-api/agents/#pagination + + Attributes: + page_size (InterpolatedString): the number of records to request, + time_field_name (InterpolatedString): the name of the to track and increment from, {: 1234} + """ + + time_field_name: Union[InterpolatedString, str] = None + + def __post_init__(self, parameters: Mapping[str, Any], **kwargs) -> None: + if not self.time_field_name: + raise ValueError("The `time_field_name` property is missing, with no-default value.") + else: + self._time_field_name = InterpolatedString.create(self.time_field_name, parameters=parameters).eval(self.config) + super().__post_init__(parameters=parameters, **kwargs) + + def should_stop_pagination(self, decoded_response: Mapping[str, Any], last_records: List[Mapping[str, Any]]) -> bool: + """ + Stop paginating when there are fewer records than the page size or the current page has no records + """ + last_records_len = len(last_records) + no_records = last_records_len == 0 + current_page_len = self._page_size.eval(self.config, response=decoded_response) + return (self._page_size and last_records_len < current_page_len) or no_records + + def get_next_page_token_offset(self, decoded_response: Mapping[str, Any]) -> int: + """ + The `records` are returned in `ASC` order. + Described in: https://developer.zendesk.com/api-reference/live-chat/chat-api/incremental_export/#incremental-agent-timeline-export + + Arguments: + decoded_response: Mapping[str, Any] -- The object with RECORDS decoded from the RESPONSE. + + Returns: + The offset value as the `next_page_token` + """ + self._offset = decoded_response[self._time_field_name] + return self._offset + + def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: + decoded_response = self.decoder.decode(response) + if self.should_stop_pagination(decoded_response, last_records): + return None + else: + return self.get_next_page_token_offset(decoded_response) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/timestamp_based_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/timestamp_based_cursor.py new file mode 100644 index 0000000000000..caab6dfc3cf90 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/components/timestamp_based_cursor.py @@ -0,0 +1,57 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from dataclasses import dataclass +from typing import Any, Mapping, MutableMapping, Optional, Union + +from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState + + +@dataclass +class ZendeskChatTimestampCursor(DatetimeBasedCursor): + """ + Override for the default `DatetimeBasedCursor` to provide the `request_params["start_time"]` with added `microseconds`, as required by the API. + More info: https://developer.zendesk.com/rest_api/docs/chat/incremental_export#incremental-agent-timeline-export + + The dates in future are not(!) allowed for the Zendesk Chat endpoints, and slicer values could be far away from exact cursor values. + + Arguments: + use_microseconds: bool - whether or not to add dummy `000000` (six zeros) to provide the microseconds unit timestamps + """ + + use_microseconds: Union[InterpolatedString, str] = True + + def __post_init__(self, parameters: Mapping[str, Any]) -> None: + self._use_microseconds = InterpolatedString.create(self.use_microseconds, parameters=parameters).eval(self.config) + self._start_date = self.config.get("start_date") + super().__post_init__(parameters=parameters) + + def add_microseconds( + self, + params: MutableMapping[str, Any], + stream_slice: Optional[StreamSlice] = None, + ) -> MutableMapping[str, Any]: + start_time = stream_slice.get(self._partition_field_start.eval(self.config)) + if start_time: + params[self.start_time_option.field_name.eval(config=self.config)] = int(start_time) * 1000000 + return params + + def get_request_params( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + params = {} + if self._use_microseconds: + params = self.add_microseconds(params, stream_slice) + else: + params[self.start_time_option.field_name.eval(config=self.config)] = stream_slice.get( + self._partition_field_start.eval(self.config) + ) + return params diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml new file mode 100644 index 0000000000000..5a5ff833c1e7a --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/manifest.yaml @@ -0,0 +1,318 @@ +version: 0.72.1 + +definitions: + # COMMON PARTS + schema_loader: + type: JsonFileSchemaLoader + file_path: "./source_zendesk_chat/schemas/{{ parameters['name'] }}.json" + selector: + description: >- + Base records selector for Full Refresh streams + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["{{ parameters.get('data_field') }}"] + authenticator: + type: BearerAuthenticator + api_token: "{{ config['credentials']['access_token'] }}" + + # PAGINATORS + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: cursor + page_size_option: + inject_into: request_parameter + type: RequestOption + field_name: limit + pagination_strategy: + type: CursorPagination + page_size: 100 + cursor_value: '{{ response.get("next_url", {}) }}' + stop_condition: '{{ not response.get("next_url", {}) }}' + paginator_id_offset: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: since_id + page_size_option: + inject_into: request_parameter + type: RequestOption + field_name: limit + pagination_strategy: + type: CustomPaginationStrategy + class_name: source_zendesk_chat.components.id_offset_pagination.ZendeskChatIdOffsetIncrementPaginationStrategy + id_field: id + page_size: 100 + paginator_time_offset: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: start_time + page_size_option: + inject_into: request_parameter + type: RequestOption + field_name: limit + pagination_strategy: + type: CustomPaginationStrategy + class_name: source_zendesk_chat.components.time_offset_pagination.ZendeskChatTimeOffsetIncrementPaginationStrategy + time_field_name: end_time + page_size: 1000 + + # REQUESTERS + requester: + description: >- + Default Base Requester for Full Refresh streams + type: HttpRequester + url_base: https://www.zopim.com/api/v2/ + path: "{{ parameters['path'] }}" + http_method: GET + authenticator: + $ref: "#/definitions/authenticator" + error_handler: + type: DefaultErrorHandler + description: >- + The default error handler + backoff_strategies: + - type: WaitTimeFromHeader + header: Retry-After + + # RETRIEVERS + retriever_base: + description: >- + Default Retriever for Full Refresh streams + record_selector: + $ref: "#/definitions/selector" + requester: + $ref: "#/definitions/requester" + paginator: + $ref: "#/definitions/paginator" + retriever_for_type_list: + $ref: "#/definitions/retriever_base" + record_selector: + $ref: "#/definitions/selector" + extractor: + type: DpathExtractor + field_path: [] + retriever_for_type_list_no_pagination: + $ref: "#/definitions/retriever_for_type_list" + paginator: + type: NoPagination + + # BASE STREAMS + base_stream: + primary_key: "id" + schema_loader: + $ref: "#/definitions/schema_loader" + retriever: + $ref: "#/definitions/retriever_base" + # FULL-REFRESH + base_stream_with_list_response_no_pagination: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/retriever_for_type_list_no_pagination" + base_stream_with_id_offset_pagination: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/retriever_for_type_list" + paginator: + $ref: "#/definitions/paginator_id_offset" + base_stream_with_time_offset_pagination: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/retriever_base" + paginator: + $ref: "#/definitions/paginator_time_offset" + requester: + $ref: "#/definitions/requester" + request_parameters: + # add `fields=(*)` to the request_params + fields: "{{ parameters['name'] + '(*)' }}" + + # INCREMENTAL + base_incremental_id_stream: + $ref: "#/definitions/base_stream_with_id_offset_pagination" + retriever: + $ref: "#/definitions/base_stream_with_id_offset_pagination/retriever" + # this is needed to ignore additional params for incremental syncs + ignore_stream_slicer_parameters_on_paginated_requests: true + incremental_sync: + type: CustomIncrementalSync + class_name: source_zendesk_chat.components.id_incremental_cursor.ZendeskChatIdIncrementalCursor + cursor_field: "id" + field_name: "since_id" + base_incremental_time_stream: + $ref: "#/definitions/base_stream_with_time_offset_pagination" + retriever: + $ref: "#/definitions/base_stream_with_time_offset_pagination/retriever" + # this is needed to ignore additional params for incremental syncs + ignore_stream_slicer_parameters_on_paginated_requests: true + incremental_sync: + type: CustomIncrementalSync + class_name: source_zendesk_chat.components.timestamp_based_cursor.ZendeskChatTimestampCursor + use_microseconds: "{{ parameters['use_microseconds'] }}" + cursor_field: "{{ parameters['cursor_field'] }}" + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%fZ" + - "%Y-%m-%dT%H:%M:%SZ" + datetime_format: "%s" + start_datetime: + datetime: "{{ format_datetime(config['start_date'], '%s') }}" + start_time_option: + field_name: start_time + inject_into: "request_parameter" + + # FULL-REFRESH STREAMS + # ACCOUNTS + accounts_stream: + description: >- + Accounts Stream: https://developer.zendesk.com/rest_api/docs/chat/accounts#show-account + $ref: "#/definitions/base_stream_with_list_response_no_pagination" + primary_key: "account_key" + $parameters: + name: "accounts" + path: "account" + # SHORTCUTS + shortcuts_stream: + description: >- + Shortcuts Stream: https://developer.zendesk.com/rest_api/docs/chat/shortcuts#list-shortcuts + $ref: "#/definitions/base_stream_with_list_response_no_pagination" + $parameters: + name: "shortcuts" + path: "shortcuts" + # ROUTING SETTINGS + routing_settings_stream: + description: >- + Routing Settings Stream: https://developer.zendesk.com/rest_api/docs/chat/routing_settings#show-account-routing-settings + $ref: "#/definitions/base_stream_with_list_response_no_pagination" + retriever: + $ref: "#/definitions/base_stream_with_list_response_no_pagination/retriever" + record_selector: + extractor: + type: DpathExtractor + field_path: ["data"] + primary_key: "" + $parameters: + name: "routing_settings" + data_field: "data" + path: "routing_settings/account" + # TRIGGERS + triggers_stream: + description: >- + Triggers Stream: https://developer.zendesk.com/rest_api/docs/chat/triggers#list-triggers + $ref: "#/definitions/base_stream_with_list_response_no_pagination" + $parameters: + name: "triggers" + path: "triggers" + # ROLES + roles_stream: + description: >- + Roles Stream: https://developer.zendesk.com/rest_api/docs/chat/roles#list-roles + $ref: "#/definitions/base_stream_with_list_response_no_pagination" + $parameters: + name: "roles" + path: "roles" + # SKILLS + skills_stream: + description: >- + Skills Stream: https://developer.zendesk.com/rest_api/docs/chat/skills#list-skills + $ref: "#/definitions/base_stream_with_list_response_no_pagination" + $parameters: + name: "skills" + path: "skills" + # GOALS + goals_stream: + description: >- + Goals Stream: https://developer.zendesk.com/rest_api/docs/chat/goals#list-goals + $ref: "#/definitions/base_stream_with_list_response_no_pagination" + $parameters: + name: "goals" + path: "goals" + # DEPARTMENTS + departments_stream: + description: >- + Departments Stream: https://developer.zendesk.com/rest_api/docs/chat/departments#list-departments + $ref: "#/definitions/base_stream_with_list_response_no_pagination" + $parameters: + name: "departments" + path: "departments" + + # INCREMENTAL STREAMS + # AGENTS + agents_stream: + description: >- + Agents Stream: https://developer.zendesk.com/rest_api/docs/chat/agents#list-agents + $ref: "#/definitions/base_incremental_id_stream" + $parameters: + name: "agents" + path: "agents" + # BANS + bans_stream: + description: >- + Bans Stream: https://developer.zendesk.com/rest_api/docs/chat/bans#list-bans + $ref: "#/definitions/base_incremental_id_stream" + retriever: + $ref: "#/definitions/base_incremental_id_stream/retriever" + record_selector: + type: RecordSelector + extractor: + type: CustomRecordExtractor + class_name: source_zendesk_chat.components.bans_record_extractor.ZendeskChatBansRecordExtractor + $parameters: + name: "bans" + path: "bans" + # AGENTS TIMELINES + agents_timelines_stream: + description: >- + Agent Timelines Stream: https://developer.zendesk.com/rest_api/docs/chat/incremental_export#incremental-agent-timeline-export + $ref: "#/definitions/base_incremental_time_stream" + transformations: + - type: AddFields + fields: + # To preserve the non-breaking changes, the additional transformations should be applied + # 1) transform the `start_time` - cursor_field to have the old datetime format == %Y-%m-%dT%H:%M:%SZ (2023-01-01T00:00:00) + - path: ["start_time"] + value: "{{ format_datetime(record.get('start_time', config['start_date']), '%Y-%m-%dT%H:%M:%SZ') }}" + # 2) make the composite `id` field + - path: ["id"] + value: "{{ record.get('agent_id', '')|string + '|' + record.get('start_time', '')|string }}" + $parameters: + cursor_field: "start_time" + name: "agent_timeline" + data_field: "agent_timeline" + path: "incremental/agent_timeline" + use_microseconds: true + # CHATS + chats_stream: + description: >- + Chats Stream: https://developer.zendesk.com/api-reference/live-chat/chat-api/incremental_export/#incremental-chat-export + $ref: "#/definitions/base_incremental_time_stream" + $parameters: + cursor_field: "update_timestamp" + name: "chats" + data_field: "chats" + path: "incremental/chats" + use_microseconds: false + +streams: + - "#/definitions/accounts_stream" + - "#/definitions/agents_stream" + - "#/definitions/agents_timelines_stream" + - "#/definitions/bans_stream" + - "#/definitions/chats_stream" + - "#/definitions/departments_stream" + - "#/definitions/goals_stream" + - "#/definitions/roles_stream" + - "#/definitions/routing_settings_stream" + - "#/definitions/shortcuts_stream" + - "#/definitions/skills_stream" + - "#/definitions/triggers_stream" + +check: + type: CheckStream + stream_names: + - routing_settings diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/schemas/agent_timeline.json b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/schemas/agent_timeline.json index 4a61d458898a5..04424877eda15 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/schemas/agent_timeline.json +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/schemas/agent_timeline.json @@ -2,6 +2,9 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { + "id": { + "type": ["null", "string"] + }, "agent_id": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py index bcad700d13d4a..2b0540f7cd8f1 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/source.py @@ -2,58 +2,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from typing import Any, Dict, List, Mapping, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from .streams import Accounts, Agents, AgentTimelines, Bans, Chats, Departments, Goals, Roles, RoutingSettings, Shortcuts, Skills, Triggers - - -class ZendeskAuthentication: - """Provides the authentication capabilities for both old and new methods.""" - - def __init__(self, config: Dict): - self.config = config - - def get_auth(self) -> TokenAuthenticator: - """Return the TokenAuthenticator object with access_token.""" - - # the old config supports for backward capability - access_token = self.config.get("access_token") - if not access_token: - # the new config supports `OAuth2.0` - access_token = self.config["credentials"]["access_token"] - - return TokenAuthenticator(token=access_token) - - -class SourceZendeskChat(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - authenticator = ZendeskAuthentication(config).get_auth() - try: - records = RoutingSettings(authenticator=authenticator).read_records(sync_mode=SyncMode.full_refresh) - next(records) - return True, None - except Exception as error: - return False, f"Unable to connect to Zendesk Chat API with the provided credentials - {error}" - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - authenticator = ZendeskAuthentication(config).get_auth() - return [ - Accounts(authenticator=authenticator), - AgentTimelines(authenticator=authenticator, start_date=config["start_date"]), - Agents(authenticator=authenticator), - Bans(authenticator=authenticator), - Chats(authenticator=authenticator, start_date=config["start_date"]), - Departments(authenticator=authenticator), - Goals(authenticator=authenticator), - Roles(authenticator=authenticator), - RoutingSettings(authenticator=authenticator), - Shortcuts(authenticator=authenticator), - Skills(authenticator=authenticator), - Triggers(authenticator=authenticator), - ] +# Declarative Source +class SourceZendeskChat(YamlDeclarativeSource): + def __init__(self) -> None: + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py b/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py deleted file mode 100644 index 353c87030e9bf..0000000000000 --- a/airbyte-integrations/connectors/source-zendesk-chat/source_zendesk_chat/streams.py +++ /dev/null @@ -1,315 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from abc import ABC, abstractmethod -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union -from urllib.parse import parse_qs, urlparse - -import pendulum -import requests -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy -from airbyte_cdk.sources.streams.http import HttpStream - - -class Stream(HttpStream, ABC): - url_base = "https://www.zopim.com/api/v2/" - primary_key = "id" - - data_field = None - - limit = 100 - - @property - def availability_strategy(self) -> Optional["AvailabilityStrategy"]: - return None - - def request_kwargs( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Mapping[str, Any]: - - return {"timeout": 60} - - def backoff_time(self, response: requests.Response) -> Optional[float]: - delay_time = response.headers.get("Retry-After") - if delay_time: - return int(delay_time) - - def path(self, **kwargs) -> str: - return self.name - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - response_data = response.json() - - if "next_url" in response_data: - next_url = response_data["next_url"] - cursor = parse_qs(urlparse(next_url).query)["cursor"] - return {"cursor": cursor} - - def request_params( - self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = {"limit": self.limit} - if next_page_token: - params.update(next_page_token) - - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - response_data = response.json() - stream_data = self.get_stream_data(response_data) - - yield from stream_data - - def get_stream_data(self, response_data: Any) -> List[dict]: - if self.data_field: - response_data = response_data.get(self.data_field, []) - - if isinstance(response_data, list): - return list(map(self.parse_response_obj, response_data)) - elif isinstance(response_data, dict): - return [self.parse_response_obj(response_data)] - else: - raise Exception(f"Unsupported type of response data for stream {self.name}") - - def parse_response_obj(self, response_obj: dict) -> dict: - return response_obj - - -class BaseIncrementalStream(Stream, ABC): - @property - @abstractmethod - def cursor_field(self) -> str: - """ - Defining a cursor field indicates that a stream is incremental, so any incremental stream must extend this class - and define a cursor field. - """ - - @abstractmethod - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object - and returning an updated state object. - """ - - @staticmethod - def _field_to_datetime(value: Union[int, str]) -> pendulum.datetime: - if isinstance(value, int): - value = pendulum.from_timestamp(value / 1000.0) - elif isinstance(value, str): - value = pendulum.parse(value) - else: - raise ValueError(f"Unsupported type of datetime field {type(value)}") - return value - - -class TimeIncrementalStream(BaseIncrementalStream, ABC): - - state_checkpoint_interval = 1000 - - def __init__(self, start_date, **kwargs): - super().__init__(**kwargs) - self._start_date = pendulum.parse(start_date) - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - response_data = response.json() - if response_data["count"] == self.limit: - return {"start_time": response_data["end_time"]} - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - latest_benchmark = self._field_to_datetime(latest_record[self.cursor_field]) - if current_stream_state.get(self.cursor_field): - state = max(latest_benchmark, self._field_to_datetime(current_stream_state[self.cursor_field])) - return {self.cursor_field: state.strftime("%Y-%m-%dT%H:%M:%SZ")} - return {self.cursor_field: latest_benchmark.strftime("%Y-%m-%dT%H:%M:%SZ")} - - def request_params( - self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, next_page_token=next_page_token) - if next_page_token: - params.update(next_page_token) - else: - start_datetime = self._start_date - if stream_state.get(self.cursor_field): - start_datetime = pendulum.parse(stream_state[self.cursor_field]) - - params.update({"start_time": int(start_datetime.timestamp())}) - - params.update({"fields": f"{self.name}(*)"}) - return params - - def path(self, **kwargs) -> str: - return f"incremental/{self.name}" - - def parse_response_obj(self, response_obj: dict) -> dict: - response_obj[self.cursor_field] = pendulum.parse(response_obj[self.cursor_field]).strftime("%Y-%m-%dT%H:%M:%SZ") - return response_obj - - -class IdIncrementalStream(BaseIncrementalStream): - cursor_field = "id" - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - latest_benchmark = latest_record[self.cursor_field] - if current_stream_state.get(self.cursor_field): - return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])} - return {self.cursor_field: latest_benchmark} - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - stream_data = self.get_stream_data(response.json()) - if len(stream_data) == self.limit: - last_object_id = stream_data[-1]["id"] - return {"since_id": last_object_id} - - def request_params( - self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, next_page_token=next_page_token) - - if next_page_token: - params.update(next_page_token) - elif stream_state.get(self.cursor_field): - params.update({"since_id": stream_state[self.cursor_field]}) - - return params - - -class Agents(IdIncrementalStream): - """ - Agents Stream: https://developer.zendesk.com/rest_api/docs/chat/agents#list-agents - """ - - -class AgentTimelines(TimeIncrementalStream): - """ - Agent Timelines Stream: https://developer.zendesk.com/rest_api/docs/chat/incremental_export#incremental-agent-timeline-export - """ - - primary_key = None - cursor_field = "start_time" - data_field = "agent_timeline" - name = "agent_timeline" - limit = 1000 - - def request_params(self, **kwargs) -> MutableMapping[str, Any]: - params = super().request_params(**kwargs) - if not kwargs.get("next_page_token"): - params["start_time"] = params["start_time"] * 1000000 - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - response_data = response.json() - stream_data = self.get_stream_data(response_data) - - def generate_key(record): - record.update({"id": "|".join((str(record.get("agent_id", "")), str(record.get("start_time", ""))))}) - return record - - # associate the surrogate key - yield from map( - generate_key, - stream_data, - ) - - -class Accounts(Stream): - """ - Accounts Stream: https://developer.zendesk.com/rest_api/docs/chat/accounts#show-account - """ - - primary_key = "account_key" - - def path(self, **kwargs) -> str: - return "account" - - -class Chats(TimeIncrementalStream): - """ - Chats Stream: https://developer.zendesk.com/api-reference/live-chat/chat-api/incremental_export/#incremental-chat-export - """ - - cursor_field = "update_timestamp" - data_field = "chats" - limit = 1000 - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - response_data = response.json() - if response_data["count"] == self.limit: - next_page = {"start_time": response_data["end_time"]} - - start_id = response_data.get("end_id") - if start_id: - next_page.update({"start_id": start_id}) - - return next_page - - -class Shortcuts(Stream): - """ - Shortcuts Stream: https://developer.zendesk.com/rest_api/docs/chat/shortcuts#list-shortcuts - """ - - -class Triggers(Stream): - """ - Triggers Stream: https://developer.zendesk.com/rest_api/docs/chat/triggers#list-triggers - """ - - -class Bans(IdIncrementalStream): - """ - Bans Stream: https://developer.zendesk.com/rest_api/docs/chat/bans#list-bans - """ - - def get_stream_data(self, response_data) -> List[dict]: - bans = response_data["ip_address"] + response_data["visitor"] - bans = sorted(bans, key=lambda x: pendulum.parse(x["created_at"]) if x["created_at"] else pendulum.datetime(1970, 1, 1)) - return bans - - -class Departments(Stream): - """ - Departments Stream: https://developer.zendesk.com/rest_api/docs/chat/departments#list-departments - """ - - -class Goals(Stream): - """ - Goals Stream: https://developer.zendesk.com/rest_api/docs/chat/goals#list-goals - """ - - -class Skills(Stream): - """ - Skills Stream: https://developer.zendesk.com/rest_api/docs/chat/skills#list-skills - """ - - -class Roles(Stream): - """ - Roles Stream: https://developer.zendesk.com/rest_api/docs/chat/roles#list-roles - """ - - -class RoutingSettings(Stream): - """ - Routing Settings Stream: https://developer.zendesk.com/rest_api/docs/chat/routing_settings#show-account-routing-settings - """ - - primary_key = "" - - name = "routing_settings" - data_field = "data" - - def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> str: - return "routing_settings/account" diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/__init__.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/__init__.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/conftest.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/conftest.py new file mode 100644 index 0000000000000..c48196cfa1edd --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/conftest.py @@ -0,0 +1,65 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, List, Mapping + +import pytest + + +@pytest.fixture +def config() -> Mapping[str, Any]: + return { + "start_date": "2020-10-01T00:00:00Z", + "subdomain": "", + "credentials": { + "credentials": "access_token", + "access_token": "__access_token__" + } + } + + +@pytest.fixture +def bans_stream_record() -> Mapping[str, Any]: + return { + "ip_address": [ + { + "reason": "test", + "type": "ip_address", + "id": 1234, + "created_at": "2021-04-21T14:42:46Z", + "ip_address": "0.0.0.0" + } + ], + "visitor": [ + { + "type": "visitor", + "id": 4444, + "visitor_name": "Visitor 4444", + "visitor_id": "visitor_id", + "reason": "test", + "created_at": "2021-04-27T13:25:01Z" + } + ] + } + + +@pytest.fixture +def bans_stream_record_extractor_expected_output() -> List[Mapping[str, Any]]: + return [ + { + "reason": "test", + "type": "ip_address", + "id": 1234, + "created_at": "2021-04-21T14:42:46Z", + "ip_address": "0.0.0.0" + }, + { + "type": "visitor", + "id": 4444, + "visitor_name": "Visitor 4444", + "visitor_id": "visitor_id", + "reason": "test", + "created_at": "2021-04-27T13:25:01Z" + }, + ] diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_bans_record_extractor.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_bans_record_extractor.py new file mode 100644 index 0000000000000..446bcc8f63dec --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_bans_record_extractor.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import requests +from source_zendesk_chat.components.bans_record_extractor import ZendeskChatBansRecordExtractor + + +def test_bans_stream_record_extractor( + requests_mock, + bans_stream_record, + bans_stream_record_extractor_expected_output, +) -> None: + test_url = "https://www.zopim.com/api/v2/bans" + requests_mock.get(test_url, json=bans_stream_record) + test_response = requests.get(test_url) + assert ZendeskChatBansRecordExtractor().extract_records(test_response) == bans_stream_record_extractor_expected_output diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py new file mode 100644 index 0000000000000..9557a312b6359 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_incremental_cursor.py @@ -0,0 +1,114 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest +from source_zendesk_chat.components.id_incremental_cursor import ZendeskChatIdIncrementalCursor + + +def _get_cursor(config) -> ZendeskChatIdIncrementalCursor: + return ZendeskChatIdIncrementalCursor( + config = config, + cursor_field = "id", + field_name = "since_id", + parameters = {}, + ) + + +@pytest.mark.parametrize( + "stream_state, expected_cursor_value, expected_state_value", + [ + ({"id": 10}, 10, {'id': 10}), + ], + ids=[ + "SET Initial State and GET State" + ] +) +def test_id_incremental_cursor_set_initial_state_and_get_stream_state( + config, + stream_state, + expected_cursor_value, + expected_state_value, +) -> None: + cursor = _get_cursor(config) + cursor.set_initial_state(stream_state) + assert cursor._cursor == expected_cursor_value + assert cursor._state == expected_cursor_value + assert cursor.get_stream_state() == expected_state_value + + +@pytest.mark.parametrize( + "test_record, expected", + [ + ({"id": 123}, 123), + ({"id": 456}, 456), + ], + ids=[ + "first", + "second" + ] +) +def test_id_incremental_cursor_close_slice(config, test_record, expected) -> None: + cursor = _get_cursor(config) + cursor.observe(stream_slice={}, record=test_record) + cursor.close_slice(stream_slice={}) + assert cursor._cursor == expected + + +@pytest.mark.parametrize( + "stream_state, input_slice, expected", + [ + ({}, {"id": 1}, {}), + ({"id": 2}, {"id": 1}, {"since_id": 2}), + ], + ids=[ + "No State", + "With State" + ] +) +def test_id_incremental_cursor_get_request_params(config, stream_state, input_slice, expected) -> None: + cursor = _get_cursor(config) + if stream_state: + cursor.set_initial_state(stream_state) + assert cursor.get_request_params(stream_slice=input_slice) == expected + + +@pytest.mark.parametrize( + "stream_state, record, expected", + [ + ({}, {"id": 1}, True), + ({"id": 2}, {"id": 1}, False), + ({"id": 2}, {"id": 3}, True), + ], + ids=[ + "No State", + "With State > Record value", + "With State < Record value", + ] +) +def test_id_incremental_cursor_should_be_synced(config, stream_state, record, expected) -> None: + cursor = _get_cursor(config) + if stream_state: + cursor.set_initial_state(stream_state) + assert cursor.should_be_synced(record=record) == expected + + +@pytest.mark.parametrize( + "first_record, second_record, expected", + [ + ({"id": 2}, {"id": 1}, True), + ({"id": 2}, {"id": 3}, False), + ({"id": 3}, {}, True), + ({}, {}, False), + ], + ids=[ + "First > Second - should synced", + "First < Second - should not be synced", + "Has First but no Second - should be synced", + "Has no First and has no Second - should not be synced", + ] +) +def test_id_incremental_cursor_is_greater_than_or_equal(config, first_record, second_record, expected) -> None: + cursor = _get_cursor(config) + assert cursor.is_greater_than_or_equal(first=first_record, second=second_record) == expected diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_offset_pagination.py new file mode 100644 index 0000000000000..5c5f4dd46b1ad --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_id_offset_pagination.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest +import requests +from source_zendesk_chat.components.id_offset_pagination import ZendeskChatIdOffsetIncrementPaginationStrategy + + +def _get_paginator(config, id_field) -> ZendeskChatIdOffsetIncrementPaginationStrategy: + return ZendeskChatIdOffsetIncrementPaginationStrategy( + config = config, + page_size = 1, + id_field = id_field, + parameters = {}, + ) + + +@pytest.mark.parametrize( + "id_field, last_records, expected", + [ + ("id", [{"id": 1}], 2), + ("id", [], None) + ], +) +def test_id_offset_increment_pagination_next_page_token(requests_mock, config, id_field, last_records, expected) -> None: + paginator = _get_paginator(config, id_field) + test_url = "https://www.zopim.com/api/v2/agents" + requests_mock.get(test_url, json=last_records) + test_response = requests.get(test_url) + assert paginator.next_page_token(test_response, last_records) == expected diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_time_offset_pagination.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_time_offset_pagination.py new file mode 100644 index 0000000000000..086ea195fac2d --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_time_offset_pagination.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest +import requests +from source_zendesk_chat.components.time_offset_pagination import ZendeskChatTimeOffsetIncrementPaginationStrategy + + +def _get_paginator(config, time_field_name) -> ZendeskChatTimeOffsetIncrementPaginationStrategy: + return ZendeskChatTimeOffsetIncrementPaginationStrategy( + config = config, + page_size = 1, + time_field_name = time_field_name, + parameters = {}, + ) + + +@pytest.mark.parametrize( + "time_field_name, response, last_records, expected", + [ + ("end_time", {"chats":[{"update_timestamp": 1}], "end_time": 2}, [{"update_timestamp": 1}], 2), + ("end_time", {"chats":[], "end_time": 3}, [], None), + ], +) +def test_time_offset_increment_pagination_next_page_token(requests_mock, config, time_field_name, response, last_records, expected) -> None: + paginator = _get_paginator(config, time_field_name) + test_url = "https://www.zopim.com/api/v2/chats" + requests_mock.get(test_url, json=response) + test_response = requests.get(test_url) + assert paginator.next_page_token(test_response, last_records) == expected diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py new file mode 100644 index 0000000000000..a98cc8283e930 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/components/test_timestamp_based_cursor.py @@ -0,0 +1,54 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest +from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType +from source_zendesk_chat.components.timestamp_based_cursor import ZendeskChatTimestampCursor + + +def _get_cursor(config, cursor_field, use_microseconds) -> ZendeskChatTimestampCursor: + cursor = ZendeskChatTimestampCursor( + start_datetime = "2020-10-01T00:00:00Z", + cursor_field = cursor_field, + datetime_format = "%s", + config = config, + parameters = {}, + use_microseconds = f"{{{ {use_microseconds} }}}", + ) + # patching missing parts + cursor.start_time_option = RequestOption( + field_name = cursor_field, + inject_into = RequestOptionType.request_parameter, + parameters={}, + ) + return cursor + + +@pytest.mark.parametrize( + "use_microseconds, input_slice, expected", + [ + (True, {"start_time": 1}, {'start_time': 1000000}), + ], +) +def test_timestamp_based_cursor_add_microseconds(config, use_microseconds, input_slice, expected) -> None: + cursor = _get_cursor(config, "start_time", use_microseconds) + test_result = cursor.add_microseconds({}, input_slice) + assert test_result == expected + + +@pytest.mark.parametrize( + "use_microseconds, input_slice, expected", + [ + (True, {"start_time": 1}, {'start_time': 1000000}), + (False, {"start_time": 1}, {'start_time': 1}), + ], + ids=[ + "WITH `use_microseconds`", + "WITHOUT `use_microseconds`", + ] +) +def test_timestamp_based_cursor_get_request_params(config, use_microseconds, input_slice, expected) -> None: + cursor = _get_cursor(config, "start_time", use_microseconds) + assert cursor.get_request_params(stream_slice=input_slice) == expected diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_source.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_source.py deleted file mode 100644 index 4607e132314f1..0000000000000 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_source.py +++ /dev/null @@ -1,76 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from unittest.mock import patch - -import pytest -import requests -from airbyte_cdk import AirbyteLogger -from source_zendesk_chat.source import SourceZendeskChat, ZendeskAuthentication -from source_zendesk_chat.streams import ( - Accounts, - Agents, - AgentTimelines, - Bans, - Chats, - Departments, - Goals, - Roles, - RoutingSettings, - Shortcuts, - Skills, - Triggers, -) - -TEST_CONFIG: dict = { - "start_date": "2020-10-01T00:00:00Z", - "access_token": "access_token", -} -TEST_INSTANCE: SourceZendeskChat = SourceZendeskChat() - - -def test_get_auth(): - expected = {"Authorization": "Bearer access_token"} - result = ZendeskAuthentication(TEST_CONFIG).get_auth().get_auth_header() - assert expected == result - - -@pytest.mark.parametrize( - "response, check_passed", - [ - (iter({"id": 123}), True), - (requests.HTTPError(), False), - ], - ids=["Success", "Fail"], -) -def test_check(response, check_passed): - with patch.object(RoutingSettings, "read_records", return_value=response) as mock_method: - result = TEST_INSTANCE.check_connection(logger=AirbyteLogger, config=TEST_CONFIG) - mock_method.assert_called() - assert check_passed == result[0] - - -@pytest.mark.parametrize( - "stream_cls", - [ - (Accounts), - (Agents), - (AgentTimelines), - (Bans), - (Chats), - (Departments), - (Goals), - (Roles), - (RoutingSettings), - (Shortcuts), - (Skills), - (Triggers), - ], -) -def test_streams(stream_cls): - streams = TEST_INSTANCE.streams(config=TEST_CONFIG) - for stream in streams: - if stream_cls in streams: - assert isinstance(stream, stream_cls) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_streams.py deleted file mode 100644 index b90941b01c72f..0000000000000 --- a/airbyte-integrations/connectors/source-zendesk-chat/unit_tests/test_streams.py +++ /dev/null @@ -1,344 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest -import requests -from source_zendesk_chat.source import ZendeskAuthentication -from source_zendesk_chat.streams import ( - Accounts, - Agents, - AgentTimelines, - Bans, - Chats, - Departments, - Goals, - Roles, - RoutingSettings, - Shortcuts, - Skills, - Triggers, -) - -TEST_CONFIG: dict = { - "start_date": "2020-10-01T00:00:00Z", - "access_token": "access_token", -} -TEST_CONFIG.update(**{"authenticator": ZendeskAuthentication(TEST_CONFIG).get_auth()}) - - -class TestFullRefreshStreams: - """ - STREAMS: - Accounts, Shortcuts, Triggers, Departments, Goals, Skills, Roles, RoutingSettings - """ - - @pytest.mark.parametrize( - "stream_cls", - [ - (Accounts), - (Departments), - (Goals), - (Roles), - (RoutingSettings), - (Shortcuts), - (Skills), - (Triggers), - ], - ) - def test_request_kwargs(self, stream_cls): - stream = stream_cls(TEST_CONFIG) - expected = {"timeout": 60} - assert expected == stream.request_kwargs(stream_state=None) - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (Accounts, "5"), - (Departments, "5"), - (Goals, "5"), - (Roles, "3"), - (RoutingSettings, "3"), - (Shortcuts, "3"), - (Skills, "1"), - (Triggers, "1"), - ], - ) - def test_backoff_time(self, requests_mock, stream_cls, expected): - stream = stream_cls(TEST_CONFIG) - url = f"{stream.url_base}{stream.path()}" - test_headers = {"Retry-After": expected} - requests_mock.get(url, headers=test_headers) - response = requests.get(url) - result = stream.backoff_time(response) - assert result == int(expected) - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (Accounts, "account"), - (Departments, "departments"), - (Goals, "goals"), - (Roles, "roles"), - (RoutingSettings, "routing_settings/account"), - (Shortcuts, "shortcuts"), - (Skills, "skills"), - (Triggers, "triggers"), - ], - ) - def test_path(self, stream_cls, expected): - stream = stream_cls(TEST_CONFIG) - result = stream.path() - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, expected_cursor", - [ - (Accounts, "MTU4MD"), - (Departments, "c1Mzc"), - (Goals, "wfHw0MzJ8"), - (Roles, "0MzJ8"), - (RoutingSettings, "MTUC4wJ8"), - (Shortcuts, "MTU4MD"), - (Skills, "c1Mzc"), - (Triggers, "0MzJ8"), - ], - ) - def test_next_page_token(self, requests_mock, stream_cls, expected_cursor): - stream = stream_cls(TEST_CONFIG) - url = f"{stream.url_base}{stream.path()}" - next_url = f"{url}/cursor.json?cursor={expected_cursor}" - test_response = {"next_url": next_url} - requests_mock.get(url, json=test_response) - response = requests.get(url) - result = stream.next_page_token(response) - assert result == {"cursor": [expected_cursor]} - - @pytest.mark.parametrize( - "stream_cls, next_page_token, expected", - [ - (Accounts, {"cursor": "MTU4MD"}, {"limit": 100, "cursor": "MTU4MD"}), - (Departments, {"cursor": "c1Mzc"}, {"limit": 100, "cursor": "c1Mzc"}), - (Goals, {"cursor": "wfHw0MzJ8"}, {"limit": 100, "cursor": "wfHw0MzJ8"}), - (Roles, {"cursor": "0MzJ8"}, {"limit": 100, "cursor": "0MzJ8"}), - (RoutingSettings, {"cursor": "MTUC4wJ8"}, {"limit": 100, "cursor": "MTUC4wJ8"}), - (Shortcuts, {"cursor": "MTU4MD"}, {"limit": 100, "cursor": "MTU4MD"}), - (Skills, {"cursor": "c1Mzc"}, {"limit": 100, "cursor": "c1Mzc"}), - (Triggers, {"cursor": "0MzJ8"}, {"limit": 100, "cursor": "0MzJ8"}), - ], - ) - def test_request_params(self, stream_cls, next_page_token, expected): - stream = stream_cls(TEST_CONFIG) - result = stream.request_params(stream_state=None, next_page_token=next_page_token) - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, test_response, expected", - [ - (Accounts, [{"id": "123"}], [{"id": "123"}]), - (Departments, {"id": "123"}, [{"id": "123"}]), - (Goals, {}, [{}]), - (Roles, [{"id": "123"}], [{"id": "123"}]), - (RoutingSettings, {"data": {"id": "123"}}, [{"id": "123"}]), - (Shortcuts, [{"id": "123"}], [{"id": "123"}]), - (Skills, [{"id": "123"}], [{"id": "123"}]), - (Triggers, [{"id": "123"}], [{"id": "123"}]), - ], - ) - def test_parse_response(self, requests_mock, stream_cls, test_response, expected): - stream = stream_cls(TEST_CONFIG) - url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json=test_response) - response = requests.get(url) - result = stream.parse_response(response) - assert list(result) == expected - - -class TestTimeIncrementalStreams: - """ - STREAMS: - AgentTimelines, Chats - """ - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (AgentTimelines, 1000), - (Chats, 1000), - ], - ) - def test_state_checkpoint_interval(self, stream_cls, expected): - stream = stream_cls(start_date=TEST_CONFIG["start_date"]) - result = stream.state_checkpoint_interval - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (AgentTimelines, "start_time"), - (Chats, "update_timestamp"), - ], - ) - def test_cursor_field(self, stream_cls, expected): - stream = stream_cls(start_date=TEST_CONFIG["start_date"]) - result = stream.cursor_field - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, test_response, expected", - [ - (AgentTimelines, {"end_time": "123"}, {"start_time": "123"}), - (Chats, {"end_time": "123"}, {"start_time": "123"}), - ], - ) - def test_next_page_token(self, requests_mock, stream_cls, test_response, expected): - stream = stream_cls(start_date=TEST_CONFIG["start_date"]) - test_response.update(**{"count": stream.limit}) - url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json=test_response) - response = requests.get(url) - result = stream.next_page_token(response) - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, current_state, last_record, expected", - [ - (AgentTimelines, {}, {"start_time": "2021-01-01"}, {"start_time": "2021-01-01T00:00:00Z"}), - (Chats, {"update_timestamp": "2022-02-02"}, {"update_timestamp": "2022-03-03"}, {"update_timestamp": "2022-03-03T00:00:00Z"}), - ], - ) - def test_get_updated_state(self, stream_cls, current_state, last_record, expected): - stream = stream_cls(start_date=TEST_CONFIG["start_date"]) - result = stream.get_updated_state(current_state, last_record) - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, stream_state, next_page_token, expected", - [ - (AgentTimelines, {}, {"start_time": "123"}, {"limit": 1000, "start_time": "123", "fields": "agent_timeline(*)"}), - (Chats, {"update_timestamp": "2022-02-02"}, {"start_time": "234"}, {"limit": 1000, "start_time": "234", "fields": "chats(*)"}), - ], - ) - def test_request_params(self, stream_cls, stream_state, next_page_token, expected): - stream = stream_cls(start_date=TEST_CONFIG["start_date"]) - result = stream.request_params(stream_state=stream_state, next_page_token=next_page_token) - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, test_response, expected", - [ - ( - AgentTimelines, - {"agent_timeline": {"id": "123", "agent_id": "test_id", "start_time": "2021-01-01"}}, - [{"id": "test_id|2021-01-01T00:00:00Z", "agent_id": "test_id", "start_time": "2021-01-01T00:00:00Z"}], - ), - ( - Chats, - {"chats": {"id": "234", "agent_id": "test_id", "update_timestamp": "2022-01-01"}}, - [{"id": "234", "agent_id": "test_id", "update_timestamp": "2022-01-01T00:00:00Z"}], - ), - ], - ) - def test_parse_response(self, requests_mock, stream_cls, test_response, expected): - stream = stream_cls(start_date=TEST_CONFIG["start_date"]) - url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json=test_response) - response = requests.get(url) - result = stream.parse_response(response) - assert list(result) == expected - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (AgentTimelines, "incremental/agent_timeline"), - (Chats, "incremental/chats"), - ], - ) - def test_path(self, stream_cls, expected): - stream = stream_cls(start_date=TEST_CONFIG["start_date"]) - result = stream.path() - assert result == expected - - -class TestIdIncrementalStreams: - """ - STREAMS: - Agents, Bans - """ - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (Agents, "agents"), - (Bans, "bans"), - ], - ) - def test_path(self, stream_cls, expected): - stream = stream_cls(TEST_CONFIG) - result = stream.path() - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (Agents, "id"), - (Bans, "id"), - ], - ) - def test_cursor_field(self, stream_cls, expected): - stream = stream_cls(TEST_CONFIG) - result = stream.cursor_field - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, current_state, last_record, expected", - [ - (Agents, {}, {"id": "1"}, {"id": "1"}), - (Bans, {"id": "1"}, {"id": "2"}, {"id": "2"}), - ], - ) - def test_get_updated_state(self, stream_cls, current_state, last_record, expected): - stream = stream_cls(TEST_CONFIG) - result = stream.get_updated_state(current_state, last_record) - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, test_response, expected", - [ - (Agents, [{"id": "2"}], {"since_id": "2"}), - ], - ) - def test_next_page_token(self, requests_mock, stream_cls, test_response, expected): - stream = stream_cls(TEST_CONFIG) - stream.limit = 1 - url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json=test_response) - response = requests.get(url) - result = stream.next_page_token(response) - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, test_response, expected", - [ - (Agents, {"id": "2"}, [{"id": "2"}]), - ], - ) - def test_parse_response(self, requests_mock, stream_cls, test_response, expected): - stream = stream_cls(TEST_CONFIG) - url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json=test_response) - response = requests.get(url) - result = stream.parse_response(response) - assert list(result) == expected - - @pytest.mark.parametrize( - "stream_cls, stream_state, next_page_token, expected", - [ - (Agents, {}, {"since_id": "1"}, {"limit": 100, "since_id": "1"}), - (Bans, {"id": "1"}, {"since_id": "2"}, {"limit": 100, "since_id": "2"}), - ], - ) - def test_request_params(self, stream_cls, stream_state, next_page_token, expected): - stream = stream_cls(TEST_CONFIG) - result = stream.request_params(stream_state=stream_state, next_page_token=next_page_token) - assert result == expected diff --git a/docs/integrations/sources/zendesk-chat.md b/docs/integrations/sources/zendesk-chat.md index 1baf884155190..ef641f77cdf7a 100644 --- a/docs/integrations/sources/zendesk-chat.md +++ b/docs/integrations/sources/zendesk-chat.md @@ -80,6 +80,7 @@ The connector is restricted by Zendesk's [requests limitation](https://developer | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------- | +| 0.3.0 | 2024-03-07 | [35867](https://github.com/airbytehq/airbyte/pull/35867) | Migrated to `YamlDeclarativeSource (Low-code)` Airbyte CDK | | 0.2.2 | 2024-02-12 | [35185](https://github.com/airbytehq/airbyte/pull/35185) | Manage dependencies with Poetry. | | 0.2.1 | 2023-10-20 | [31643](https://github.com/airbytehq/airbyte/pull/31643) | Upgrade base image to airbyte/python-connector-base:1.1.0 | | 0.2.0 | 2023-10-11 | [30526](https://github.com/airbytehq/airbyte/pull/30526) | Use the python connector base image, remove dockerfile and implement build_customization.py |