diff --git a/airbyte-integrations/connectors/source-slack/.coveragerc b/airbyte-integrations/connectors/source-slack/.coveragerc new file mode 100644 index 000000000000..ce3230114471 --- /dev/null +++ b/airbyte-integrations/connectors/source-slack/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + source_slack/run.py diff --git a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml index 2fd1ba9af04b..a9ee3dea68d4 100644 --- a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml @@ -6,7 +6,9 @@ acceptance_tests: - spec_path: "source_slack/spec.json" backward_compatibility_tests_config: # edited `min`/`max` > `minimum`/`maximum` for `lookback_window` field - disable_for_version: "0.1.26" + #disable_for_version: "0.1.26" + # slight changes: removed doc url, added new null oauth param + disable_for_version: "0.3.10" connection: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-slack/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-slack/integration_tests/abnormal_state.json index d55652e4e69a..104b5856e074 100644 --- a/airbyte-integrations/connectors/source-slack/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-slack/integration_tests/abnormal_state.json @@ -9,8 +9,49 @@ { "type": "STREAM", "stream": { - "stream_state": { "float_ts": 7270247822 }, - "stream_descriptor": { "name": "channel_messages" } + "stream_descriptor": { + "name": "channel_messages" + }, + "stream_state": { + "states": [ + { + "partition": { + "channel_id": "C04LTCM2Y56", + "parent_slice": {} + }, + "cursor": { + "float_ts": "2534945416" + } + }, + { + "partition": { + "channel": "C04KX3KEZ54", + "parent_slice": {} + }, + "cursor": { + "float_ts": "2534945416" + } + }, + { + "partition": { + "channel": "C04L3M4PTJ6", + "parent_slice": {} + }, + "cursor": { + "float_ts": "2534945416" + } + }, + { + "partition": { + "channel": "C04LTCM2Y56", + "parent_slice": {} + }, + "cursor": { + "float_ts": "2534945416" + } + } + ] + } } } ] diff --git a/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.jsonl index e5b6c7911376..6ed8e6208f04 100644 --- a/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.jsonl @@ -4,12 +4,14 @@ {"stream": "channel_members", "data": {"member_id": "U04L65GPMKN", "channel_id": "C04KX3KEZ54"}, "emitted_at": 1707568736171} {"stream": "channel_members", "data": {"member_id": "U04LY6NARHU", "channel_id": "C04KX3KEZ54"}, "emitted_at": 1707568736172} {"stream": "channel_members", "data": {"member_id": "U04M23SBJGM", "channel_id": "C04KX3KEZ54"}, "emitted_at": 1707568736172} -{"stream": "channel_messages", "data": {"client_msg_id": "3ae60d35-58b8-441c-923a-75de35a4ed8a", "type": "message", "text": "Test Thread 2", "user": "U04L65GPMKN", "ts": "1683104542.931169", "blocks": [{"type": "rich_text", "block_id": "WLB", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 2"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104568.059569", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104568.059569", "channel_id": "C04KX3KEZ54", "float_ts": 1683104542.931169}, "emitted_at": 1707568738170} -{"stream": "channel_messages", "data": {"client_msg_id": "e27672c0-451e-42a6-8eff-a14d2db8ac1e", "type": "message", "text": "Test Thread 1", "user": "U04L65GPMKN", "ts": "1683104499.808709", "blocks": [{"type": "rich_text", "block_id": "0j7", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 1"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104528.084359", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104528.084359", "channel_id": "C04LTCM2Y56", "float_ts": 1683104499.808709}, "emitted_at": 1707569060525} -{"stream": "channel_messages", "data": {"type": "message", "subtype": "reminder_add", "text": " set up a reminder \u201ctest reminder\u201d in this channel at 9AM tomorrow, Eastern European Summer Time.", "user": "U04L65GPMKN", "ts": "1695814864.744249", "channel_id": "C04LTCM2Y56", "float_ts": 1695814864.744249}, "emitted_at": 1707569208689} -{"stream": "threads", "data": {"client_msg_id": "3ae60d35-58b8-441c-923a-75de35a4ed8a", "type": "message", "text": "Test Thread 2", "user": "U04L65GPMKN", "ts": "1683104542.931169", "blocks": [{"type": "rich_text", "block_id": "WLB", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 2"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104568.059569", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104568.059569", "channel_id": "C04KX3KEZ54", "float_ts": 1683104542.931169}, "emitted_at": 1707569354932} -{"stream": "threads", "data": {"client_msg_id": "3e96d351-270c-493f-a1a0-fdc3c4c0e11f", "type": "message", "text": "<@U04M23SBJGM> test test test", "user": "U04L65GPMKN", "ts": "1683104559.922849", "blocks": [{"type": "rich_text", "block_id": "tX6vr", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04M23SBJGM"}, {"type": "text", "text": " test test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "parent_user_id": "U04L65GPMKN", "channel_id": "C04KX3KEZ54", "float_ts": 1683104559.922849}, "emitted_at": 1707569354933} -{"stream": "threads", "data": {"client_msg_id": "08023e44-9d18-41ed-81dd-5f04ed699656", "type": "message", "text": "<@U04LY6NARHU> test test", "user": "U04L65GPMKN", "ts": "1683104568.059569", "blocks": [{"type": "rich_text", "block_id": "IyUF", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04LY6NARHU"}, {"type": "text", "text": " test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "parent_user_id": "U04L65GPMKN", "channel_id": "C04KX3KEZ54", "float_ts": 1683104568.059569}, "emitted_at": 1707569354933} -{"stream": "users", "data": {"id": "USLACKBOT", "team_id": "T04KX3KDDU6", "name": "slackbot", "deleted": false, "color": "757575", "real_name": "Slackbot", "tz": "America/Los_Angeles", "tz_label": "Pacific Standard Time", "tz_offset": -28800, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Slackbot", "real_name_normalized": "Slackbot", "display_name": "Slackbot", "display_name_normalized": "Slackbot", "fields": {}, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "sv41d8cd98f0", "always_active": true, "first_name": "slackbot", "last_name": "", "image_24": "https://a.slack-edge.com/80588/img/slackbot_24.png", "image_32": "https://a.slack-edge.com/80588/img/slackbot_32.png", "image_48": "https://a.slack-edge.com/80588/img/slackbot_48.png", "image_72": "https://a.slack-edge.com/80588/img/slackbot_72.png", "image_192": "https://a.slack-edge.com/80588/marketing/img/avatars/slackbot/avatar-slackbot.png", "image_512": "https://a.slack-edge.com/80588/img/slackbot_512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 0, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1707569357949} -{"stream": "users", "data": {"id": "U04KUMXNYMV", "team_id": "T04KX3KDDU6", "name": "deactivateduser693438", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-24.png", "image_32": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-32.png", "image_48": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-48.png", "image_72": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-72.png", "image_192": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-192.png", "image_512": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090804, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1707569357951} -{"stream": "users", "data": {"id": "U04L2KY5CES", "team_id": "T04KX3KDDU6", "name": "deactivateduser686066", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-24.png", "image_32": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-32.png", "image_48": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-48.png", "image_72": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-72.png", "image_192": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-192.png", "image_512": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090785, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1707569357951} +{"stream": "channel_messages", "data": {"user": "U04L65GPMKN", "type": "message", "ts": "1683104542.931169", "client_msg_id": "3ae60d35-58b8-441c-923a-75de35a4ed8a", "text": "Test Thread 2", "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104568.059569", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104568.059569", "blocks": [{"type": "rich_text", "block_id": "WLB", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 2"}]}]}], "float_ts": 1683104542, "channel_id": "C04KX3KEZ54"}, "emitted_at": 1711041520506} +{"stream": "channel_messages", "data": {"user": "U04L65GPMKN", "type": "message", "ts": "1683104499.808709", "client_msg_id": "e27672c0-451e-42a6-8eff-a14d2db8ac1e", "text": "Test Thread 1", "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104528.084359", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104528.084359", "blocks": [{"type": "rich_text", "block_id": "0j7", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 1"}]}]}], "float_ts": 1683104499, "channel_id": "C04LTCM2Y56"}, "emitted_at": 1711041522765} +{"stream": "channel_messages", "data": {"user": "USLACKBOT", "type": "message", "ts": "1695880827.186049", "bot_id": "B01", "text": "Reminder: test reminder.", "team": "T04KX3KDDU6", "blocks": [{"type": "rich_text", "block_id": "BGzX", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Reminder: test reminder."}]}]}], "float_ts": 1695880827, "channel_id": "C04LTCM2Y56"}, "emitted_at": 1711041523073} +{"stream": "channel_messages", "data": {"subtype": "reminder_add", "user": "U04L65GPMKN", "type": "message", "ts": "1695814864.744249", "text": " set up a reminder \u201ctest reminder\u201d in this channel at 9AM tomorrow, Eastern European Summer Time.", "float_ts": 1695814864, "channel_id": "C04LTCM2Y56"}, "emitted_at": 1711041523080} +{"stream": "threads", "data": {"user": "U04L65GPMKN", "type": "message", "ts": "1683104542.931169", "client_msg_id": "3ae60d35-58b8-441c-923a-75de35a4ed8a", "text": "Test Thread 2", "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104568.059569", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104568.059569", "blocks": [{"type": "rich_text", "block_id": "WLB", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 2"}]}]}], "channel_id": "C04KX3KEZ54", "float_ts": 1683104542.931169}, "emitted_at": 1712056304168} +{"stream": "threads", "data": {"user": "U04L65GPMKN", "type": "message", "ts": "1683104559.922849", "client_msg_id": "3e96d351-270c-493f-a1a0-fdc3c4c0e11f", "text": "<@U04M23SBJGM> test test test", "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "parent_user_id": "U04L65GPMKN", "blocks": [{"type": "rich_text", "block_id": "tX6vr", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04M23SBJGM"}, {"type": "text", "text": " test test test"}]}]}], "channel_id": "C04KX3KEZ54", "float_ts": 1683104559.922849}, "emitted_at": 1712056304169} +{"stream": "threads", "data": {"user": "U04L65GPMKN", "type": "message", "ts": "1683104568.059569", "client_msg_id": "08023e44-9d18-41ed-81dd-5f04ed699656", "text": "<@U04LY6NARHU> test test", "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "parent_user_id": "U04L65GPMKN", "blocks": [{"type": "rich_text", "block_id": "IyUF", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04LY6NARHU"}, {"type": "text", "text": " test test"}]}]}], "channel_id": "C04KX3KEZ54", "float_ts": 1683104568.059569}, "emitted_at": 1712056304169} +{"stream": "threads", "data": {"user": "USLACKBOT", "type": "message", "ts": "1695880827.186049", "bot_id": "B01", "text": "Reminder: test reminder.", "team": "T04KX3KDDU6", "blocks": [{"type": "rich_text", "block_id": "BGzX", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Reminder: test reminder."}]}]}], "channel_id": "C04LTCM2Y56", "float_ts": 1695880827.186049}, "emitted_at": 1712056304703} +{"stream": "users", "data": {"id": "USLACKBOT", "team_id": "T04KX3KDDU6", "name": "slackbot", "deleted": false, "color": "757575", "real_name": "Slackbot", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Slackbot", "real_name_normalized": "Slackbot", "display_name": "Slackbot", "display_name_normalized": "Slackbot", "fields": {}, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "sv41d8cd98f0", "always_active": true, "first_name": "slackbot", "last_name": "", "image_24": "https://a.slack-edge.com/80588/img/slackbot_24.png", "image_32": "https://a.slack-edge.com/80588/img/slackbot_32.png", "image_48": "https://a.slack-edge.com/80588/img/slackbot_48.png", "image_72": "https://a.slack-edge.com/80588/img/slackbot_72.png", "image_192": "https://a.slack-edge.com/80588/marketing/img/avatars/slackbot/avatar-slackbot.png", "image_512": "https://a.slack-edge.com/80588/img/slackbot_512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 0, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1710501138877} +{"stream": "users", "data": {"id": "U04KUMXNYMV", "team_id": "T04KX3KDDU6", "name": "deactivateduser693438", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-24.png", "image_32": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-32.png", "image_48": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-48.png", "image_72": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-72.png", "image_192": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-192.png", "image_512": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090804, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1710501138879} +{"stream": "users", "data": {"id": "U04L2KY5CES", "team_id": "T04KX3KDDU6", "name": "deactivateduser686066", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-24.png", "image_32": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-32.png", "image_48": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-48.png", "image_72": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-72.png", "image_192": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-192.png", "image_512": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090785, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1710501138881} diff --git a/airbyte-integrations/connectors/source-slack/metadata.yaml b/airbyte-integrations/connectors/source-slack/metadata.yaml index a8159eb12552..37373659df83 100644 --- a/airbyte-integrations/connectors/source-slack/metadata.yaml +++ b/airbyte-integrations/connectors/source-slack/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: c2281cee-86f9-4a86-bb48-d23286b4c7bd - dockerImageTag: 0.4.1 + dockerImageTag: 1.0.0 dockerRepository: airbyte/source-slack documentationUrl: https://docs.airbyte.com/integrations/sources/slack githubIssueLabel: source-slack @@ -27,6 +27,19 @@ data: oss: enabled: true releaseStage: generally_available + releases: + breakingChanges: + 1.0.0: + message: + The source Slack connector is being migrated from the Python CDK to our declarative low-code CDK. + Due to changes in the handling of state format for incremental substreams, this migration constitutes a breaking change for the channel_messages stream. + Users will need to reset source configuration, refresh the source schema and reset the channel_messages stream after upgrading. + For more information, see our migration documentation for source Slack. + upgradeDeadline: "2024-04-29" + scopedImpact: + - scopeType: stream + impactedScopes: + - "channel_messages" suggestedStreams: streams: - users @@ -37,5 +50,5 @@ data: supportLevel: certified tags: - language:python - - cdk:python + - cdk:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-slack/poetry.lock b/airbyte-integrations/connectors/source-slack/poetry.lock index a6a77535d118..6cf0fe1c2aab 100644 --- a/airbyte-integrations/connectors/source-slack/poetry.lock +++ b/airbyte-integrations/connectors/source-slack/poetry.lock @@ -1,18 +1,18 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "airbyte-cdk" -version = "0.77.2" +version = "0.81.4" description = "A framework for writing Airbyte Connectors." optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "airbyte_cdk-0.77.2-py3-none-any.whl", hash = "sha256:6dffbe0c4b3454a5cdd20525b4f1e9cfef2e80c005b6b30473fc5bf6f75af64e"}, - {file = "airbyte_cdk-0.77.2.tar.gz", hash = "sha256:84aeb27862a18e135c7bc3a5dfc363037665d428e7495e8824673f853adcca70"}, + {file = "airbyte_cdk-0.81.4-py3-none-any.whl", hash = "sha256:4ed193da4e8be4867e1d8983172d10afb3c3b10f3e10ec618431deec1f2af4cb"}, + {file = "airbyte_cdk-0.81.4.tar.gz", hash = "sha256:5c63d8c792edf5f24d0ad804b34b3ebcc056ecede6cb4f87ebf9ac07aa987f24"}, ] [package.dependencies] -airbyte-protocol-models = "0.5.1" +airbyte-protocol-models = "*" backoff = "*" cachetools = "*" Deprecated = ">=1.2,<1.3" @@ -32,19 +32,19 @@ requests_cache = "*" wcmatch = "8.4" [package.extras] -file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pyarrow (>=15.0.0,<15.1.0)", "pytesseract (==0.3.10)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (>=15.0.0,<15.1.0)", "pytesseract (==0.3.10)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] sphinx-docs = ["Sphinx (>=4.2,<4.3)", "sphinx-rtd-theme (>=1.0,<1.1)"] vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] [[package]] name = "airbyte-protocol-models" -version = "0.5.1" +version = "0.9.0" description = "Declares the Airbyte Protocol." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, - {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, + {file = "airbyte_protocol_models-0.9.0-py3-none-any.whl", hash = "sha256:e972e140b5efd1edad5a338bcae8fdee9fc12545caf2c321e0f61b151c163a9b"}, + {file = "airbyte_protocol_models-0.9.0.tar.gz", hash = "sha256:40b69c33df23fe82d7078e84beb123bd604480e4d73cb277a890fcc92aedc8d2"}, ] [package.dependencies] @@ -300,6 +300,20 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "freezegun" +version = "1.4.0" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.7" +files = [ + {file = "freezegun-1.4.0-py3-none-any.whl", hash = "sha256:55e0fc3c84ebf0a96a5aa23ff8b53d70246479e9a68863f1fcac5a3e52f19dd6"}, + {file = "freezegun-1.4.0.tar.gz", hash = "sha256:10939b0ba0ff5adaecf3b06a5c2f73071d9678e507c5eaedb23c761d56ac774b"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + [[package]] name = "genson" version = "1.2.2" @@ -312,13 +326,13 @@ files = [ [[package]] name = "idna" -version = "3.6" +version = "3.7" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.5" files = [ - {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, - {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, + {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, + {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, ] [[package]] @@ -552,47 +566,47 @@ files = [ [[package]] name = "pydantic" -version = "1.10.14" +version = "1.10.15" description = "Data validation and settings management using python type hints" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic-1.10.14-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054"}, - {file = "pydantic-1.10.14-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87"}, - {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d"}, - {file = "pydantic-1.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e"}, - {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9"}, - {file = "pydantic-1.10.14-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a"}, - {file = "pydantic-1.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf"}, - {file = "pydantic-1.10.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7"}, - {file = "pydantic-1.10.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"}, - {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663"}, - {file = "pydantic-1.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f"}, - {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046"}, - {file = "pydantic-1.10.14-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca"}, - {file = "pydantic-1.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f"}, - {file = "pydantic-1.10.14-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c"}, - {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5"}, - {file = "pydantic-1.10.14-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c"}, - {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc"}, - {file = "pydantic-1.10.14-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe"}, - {file = "pydantic-1.10.14-cp37-cp37m-win_amd64.whl", hash = "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01"}, - {file = "pydantic-1.10.14-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee"}, - {file = "pydantic-1.10.14-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597"}, - {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee"}, - {file = "pydantic-1.10.14-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f"}, - {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022"}, - {file = "pydantic-1.10.14-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f"}, - {file = "pydantic-1.10.14-cp38-cp38-win_amd64.whl", hash = "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a"}, - {file = "pydantic-1.10.14-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4"}, - {file = "pydantic-1.10.14-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347"}, - {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7"}, - {file = "pydantic-1.10.14-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea"}, - {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f"}, - {file = "pydantic-1.10.14-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593"}, - {file = "pydantic-1.10.14-cp39-cp39-win_amd64.whl", hash = "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8"}, - {file = "pydantic-1.10.14-py3-none-any.whl", hash = "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c"}, - {file = "pydantic-1.10.14.tar.gz", hash = "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6"}, + {file = "pydantic-1.10.15-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:22ed12ee588b1df028a2aa5d66f07bf8f8b4c8579c2e96d5a9c1f96b77f3bb55"}, + {file = "pydantic-1.10.15-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:75279d3cac98186b6ebc2597b06bcbc7244744f6b0b44a23e4ef01e5683cc0d2"}, + {file = "pydantic-1.10.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50f1666a9940d3d68683c9d96e39640f709d7a72ff8702987dab1761036206bb"}, + {file = "pydantic-1.10.15-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82790d4753ee5d00739d6cb5cf56bceb186d9d6ce134aca3ba7befb1eedbc2c8"}, + {file = "pydantic-1.10.15-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:d207d5b87f6cbefbdb1198154292faee8017d7495a54ae58db06762004500d00"}, + {file = "pydantic-1.10.15-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e49db944fad339b2ccb80128ffd3f8af076f9f287197a480bf1e4ca053a866f0"}, + {file = "pydantic-1.10.15-cp310-cp310-win_amd64.whl", hash = "sha256:d3b5c4cbd0c9cb61bbbb19ce335e1f8ab87a811f6d589ed52b0254cf585d709c"}, + {file = "pydantic-1.10.15-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c3d5731a120752248844676bf92f25a12f6e45425e63ce22e0849297a093b5b0"}, + {file = "pydantic-1.10.15-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c365ad9c394f9eeffcb30a82f4246c0006417f03a7c0f8315d6211f25f7cb654"}, + {file = "pydantic-1.10.15-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3287e1614393119c67bd4404f46e33ae3be3ed4cd10360b48d0a4459f420c6a3"}, + {file = "pydantic-1.10.15-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:be51dd2c8596b25fe43c0a4a59c2bee4f18d88efb8031188f9e7ddc6b469cf44"}, + {file = "pydantic-1.10.15-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6a51a1dd4aa7b3f1317f65493a182d3cff708385327c1c82c81e4a9d6d65b2e4"}, + {file = "pydantic-1.10.15-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4e316e54b5775d1eb59187f9290aeb38acf620e10f7fd2f776d97bb788199e53"}, + {file = "pydantic-1.10.15-cp311-cp311-win_amd64.whl", hash = "sha256:0d142fa1b8f2f0ae11ddd5e3e317dcac060b951d605fda26ca9b234b92214986"}, + {file = "pydantic-1.10.15-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7ea210336b891f5ea334f8fc9f8f862b87acd5d4a0cbc9e3e208e7aa1775dabf"}, + {file = "pydantic-1.10.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3453685ccd7140715e05f2193d64030101eaad26076fad4e246c1cc97e1bb30d"}, + {file = "pydantic-1.10.15-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bea1f03b8d4e8e86702c918ccfd5d947ac268f0f0cc6ed71782e4b09353b26f"}, + {file = "pydantic-1.10.15-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:005655cabc29081de8243126e036f2065bd7ea5b9dff95fde6d2c642d39755de"}, + {file = "pydantic-1.10.15-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:af9850d98fc21e5bc24ea9e35dd80a29faf6462c608728a110c0a30b595e58b7"}, + {file = "pydantic-1.10.15-cp37-cp37m-win_amd64.whl", hash = "sha256:d31ee5b14a82c9afe2bd26aaa405293d4237d0591527d9129ce36e58f19f95c1"}, + {file = "pydantic-1.10.15-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5e09c19df304b8123938dc3c53d3d3be6ec74b9d7d0d80f4f4b5432ae16c2022"}, + {file = "pydantic-1.10.15-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7ac9237cd62947db00a0d16acf2f3e00d1ae9d3bd602b9c415f93e7a9fc10528"}, + {file = "pydantic-1.10.15-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:584f2d4c98ffec420e02305cf675857bae03c9d617fcfdc34946b1160213a948"}, + {file = "pydantic-1.10.15-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bbc6989fad0c030bd70a0b6f626f98a862224bc2b1e36bfc531ea2facc0a340c"}, + {file = "pydantic-1.10.15-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d573082c6ef99336f2cb5b667b781d2f776d4af311574fb53d908517ba523c22"}, + {file = "pydantic-1.10.15-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6bd7030c9abc80134087d8b6e7aa957e43d35714daa116aced57269a445b8f7b"}, + {file = "pydantic-1.10.15-cp38-cp38-win_amd64.whl", hash = "sha256:3350f527bb04138f8aff932dc828f154847fbdc7a1a44c240fbfff1b57f49a12"}, + {file = "pydantic-1.10.15-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:51d405b42f1b86703555797270e4970a9f9bd7953f3990142e69d1037f9d9e51"}, + {file = "pydantic-1.10.15-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a980a77c52723b0dc56640ced396b73a024d4b74f02bcb2d21dbbac1debbe9d0"}, + {file = "pydantic-1.10.15-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67f1a1fb467d3f49e1708a3f632b11c69fccb4e748a325d5a491ddc7b5d22383"}, + {file = "pydantic-1.10.15-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:676ed48f2c5bbad835f1a8ed8a6d44c1cd5a21121116d2ac40bd1cd3619746ed"}, + {file = "pydantic-1.10.15-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:92229f73400b80c13afcd050687f4d7e88de9234d74b27e6728aa689abcf58cc"}, + {file = "pydantic-1.10.15-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2746189100c646682eff0bce95efa7d2e203420d8e1c613dc0c6b4c1d9c1fde4"}, + {file = "pydantic-1.10.15-cp39-cp39-win_amd64.whl", hash = "sha256:394f08750bd8eaad714718812e7fab615f873b3cdd0b9d84e76e51ef3b50b6b7"}, + {file = "pydantic-1.10.15-py3-none-any.whl", hash = "sha256:28e552a060ba2740d0d2aabe35162652c1459a0b9069fe0db7f4ee0e18e74d58"}, + {file = "pydantic-1.10.15.tar.gz", hash = "sha256:ca832e124eda231a60a041da4f013e3ff24949d94a01154b137fc2f2a43c3ffb"}, ] [package.dependencies] @@ -837,13 +851,13 @@ yaml = ["pyyaml (>=6.0.1)"] [[package]] name = "requests-mock" -version = "1.12.0" +version = "1.12.1" description = "Mock out responses from the requests package" optional = false -python-versions = "*" +python-versions = ">=3.5" files = [ - {file = "requests-mock-1.12.0.tar.gz", hash = "sha256:4e34f2a2752f0b78397fb414526605d95fcdeab021ac1f26d18960e7eb41f6a8"}, - {file = "requests_mock-1.12.0-py2.py3-none-any.whl", hash = "sha256:4f6fdf956de568e0bac99eee4ad96b391c602e614cc0ad33e7f5c72edd699e70"}, + {file = "requests-mock-1.12.1.tar.gz", hash = "sha256:e9e12e333b525156e82a3c852f22016b9158220d2f47454de9cae8a77d371401"}, + {file = "requests_mock-1.12.1-py2.py3-none-any.whl", hash = "sha256:b1e37054004cdd5e56c84454cc7df12b25f90f382159087f4b6915aaeef39563"}, ] [package.dependencies] @@ -854,18 +868,18 @@ fixture = ["fixtures"] [[package]] name = "setuptools" -version = "69.2.0" +version = "69.5.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.2.0-py3-none-any.whl", hash = "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"}, - {file = "setuptools-69.2.0.tar.gz", hash = "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e"}, + {file = "setuptools-69.5.1-py3-none-any.whl", hash = "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32"}, + {file = "setuptools-69.5.1.tar.gz", hash = "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -892,13 +906,13 @@ files = [ [[package]] name = "typing-extensions" -version = "4.10.0" +version = "4.11.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, - {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, + {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"}, + {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"}, ] [[package]] @@ -1028,4 +1042,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9,<3.12" -content-hash = "725b28d6832d760577625b1c889cb58847a9971894c7d9276708078685c56b38" +content-hash = "59138844bec5f4f46b8a260d963d206e9881f8580ecdbeb4329d266ec0071a75" diff --git a/airbyte-integrations/connectors/source-slack/pyproject.toml b/airbyte-integrations/connectors/source-slack/pyproject.toml index a4ba01d34e60..aca63d06159f 100644 --- a/airbyte-integrations/connectors/source-slack/pyproject.toml +++ b/airbyte-integrations/connectors/source-slack/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "0.4.1" +version = "1.0.0" name = "source-slack" description = "Source implementation for Slack." authors = [ "Airbyte ",] @@ -19,6 +19,7 @@ include = "source_slack" python = "^3.9,<3.12" pendulum = "==2.1.2" airbyte-cdk = "^0" +freezegun = "^1.4.0" [tool.poetry.scripts] source-slack = "source_slack.run:run" diff --git a/airbyte-integrations/connectors/source-slack/source_slack/components/__init__.py b/airbyte-integrations/connectors/source-slack/source_slack/components/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-slack/source_slack/components/channel_members_extractor.py b/airbyte-integrations/connectors/source-slack/source_slack/components/channel_members_extractor.py new file mode 100644 index 000000000000..9dbb401a07e9 --- /dev/null +++ b/airbyte-integrations/connectors/source-slack/source_slack/components/channel_members_extractor.py @@ -0,0 +1,21 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from dataclasses import dataclass +from typing import List + +import requests +from airbyte_cdk.sources.declarative.extractors import DpathExtractor +from airbyte_cdk.sources.declarative.types import Record + + +@dataclass +class ChannelMembersExtractor(DpathExtractor): + """ + Transform response from list of strings to list dicts: + from: ['aa', 'bb'] + to: [{'member_id': 'aa'}, {{'member_id': 'bb'}] + """ + + def extract_records(self, response: requests.Response) -> List[Record]: + records = super().extract_records(response) + return [{"member_id": record} for record in records] diff --git a/airbyte-integrations/connectors/source-slack/source_slack/components/join_channels.py b/airbyte-integrations/connectors/source-slack/source_slack/components/join_channels.py new file mode 100644 index 000000000000..e7f33851784c --- /dev/null +++ b/airbyte-integrations/connectors/source-slack/source_slack/components/join_channels.py @@ -0,0 +1,123 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import logging +from functools import partial +from typing import Any, Iterable, List, Mapping, Optional + +import requests +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.declarative.partition_routers import SinglePartitionRouter +from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever +from airbyte_cdk.sources.declarative.types import Record, StreamSlice +from airbyte_cdk.sources.streams.core import StreamData +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + +LOGGER = logging.getLogger("airbyte_logger") + + +class JoinChannelsStream(HttpStream): + """ + This class is a special stream which joins channels because the Slack API only returns messages from channels this bot is in. + Its responses should only be logged for debugging reasons, not read as records. + """ + + url_base = "https://slack.com/api/" + http_method = "POST" + primary_key = "id" + + def __init__(self, channel_filter: List[str] = None, **kwargs): + self.channel_filter = channel_filter or [] + super().__init__(**kwargs) + + def path(self, **kwargs) -> str: + return "conversations.join" + + def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable: + """ + Override to simply indicate that the specific channel was joined successfully. + This method should not return any data, but should return an empty iterable. + """ + is_ok = response.json().get("ok", False) + if is_ok: + self.logger.info(f"Successfully joined channel: {stream_slice['channel_name']}") + else: + self.logger.info(f"Unable to joined channel: {stream_slice['channel_name']}. Reason: {response.json()}") + return [] + + def request_body_json(self, stream_slice: Mapping = None, **kwargs) -> Optional[Mapping]: + if stream_slice: + return {"channel": stream_slice.get("channel")} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + The pagination is not applicable to this Service Stream. + """ + return None + + +class ChannelsRetriever(SimpleRetriever): + def __post_init__(self, parameters: Mapping[str, Any]): + super().__post_init__(parameters) + self.stream_slicer = SinglePartitionRouter(parameters={}) + self.record_selector.transformations = [] + + def should_join_to_channel(self, config: Mapping[str, Any], record: Record) -> bool: + """ + The `is_member` property indicates whether the API Bot is already assigned / joined to the channel. + https://api.slack.com/types/conversation#booleans + """ + return config["join_channels"] and not record.get("is_member") + + def make_join_channel_slice(self, channel: Mapping[str, Any]) -> Mapping[str, Any]: + channel_id: str = channel.get("id") + channel_name: str = channel.get("name") + LOGGER.info(f"Joining Slack Channel: `{channel_name}`") + return {"channel": channel_id, "channel_name": channel_name} + + def join_channels_stream(self, config) -> JoinChannelsStream: + token = config["credentials"].get("api_token") or config["credentials"].get("access_token") + authenticator = TokenAuthenticator(token) + channel_filter = config["channel_filter"] + return JoinChannelsStream(authenticator=authenticator, channel_filter=channel_filter) + + def join_channel(self, config: Mapping[str, Any], record: Mapping[str, Any]): + list( + self.join_channels_stream(config).read_records( + sync_mode=SyncMode.full_refresh, + stream_slice=self.make_join_channel_slice(record), + ) + ) + + def read_records( + self, + records_schema: Mapping[str, Any], + stream_slice: Optional[StreamSlice] = None, + ) -> Iterable[StreamData]: + _slice = stream_slice or StreamSlice(partition={}, cursor_slice={}) # None-check + + self._paginator.reset() + + most_recent_record_from_slice = None + record_generator = partial( + self._parse_records, + stream_state=self.state or {}, + stream_slice=_slice, + records_schema=records_schema, + ) + + for stream_data in self._read_pages(record_generator, self.state, _slice): + # joining channel logic + if self.should_join_to_channel(self.config, stream_data): + self.join_channel(self.config, stream_data) + + current_record = self._extract_record(stream_data, _slice) + if self.cursor and current_record: + self.cursor.observe(_slice, current_record) + + most_recent_record_from_slice = self._get_most_recent_record(most_recent_record_from_slice, current_record, _slice) + yield stream_data + + if self.cursor: + self.cursor.observe(_slice, most_recent_record_from_slice) + return diff --git a/airbyte-integrations/connectors/source-slack/source_slack/config_migrations.py b/airbyte-integrations/connectors/source-slack/source_slack/config_migrations.py new file mode 100644 index 000000000000..cc6d9cd03607 --- /dev/null +++ b/airbyte-integrations/connectors/source-slack/source_slack/config_migrations.py @@ -0,0 +1,73 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import logging +from typing import Any, List, Mapping + +from airbyte_cdk import AirbyteEntrypoint +from airbyte_cdk.config_observation import create_connector_config_control_message +from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository +from source_slack import SourceSlack + +logger = logging.getLogger("airbyte_logger") + + +class MigrateLegacyConfig: + message_repository: MessageRepository = InMemoryMessageRepository() + + @classmethod + def _should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + legacy config: + { + "start_date": "2021-07-22T20:00:00Z", + "end_date": "2021-07-23T20:00:00Z", + "lookback_window": 1, + "join_channels": True, + "channel_filter": ["airbyte-for-beginners", "good-reads"], + "api_token": "api-token" + } + api token should be in the credentials object + """ + if config.get("api_token") and not config.get("credentials"): + return True + return False + + @classmethod + def _move_token_to_credentials(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: + api_token = config["api_token"] + config.update({"credentials": {"api_token": api_token, "option_title": "API Token Credentials"}}) + config.pop("api_token") + return config + + @classmethod + def _modify_and_save(cls, config_path: str, source: SourceSlack, config: Mapping[str, Any]) -> Mapping[str, Any]: + migrated_config = cls._move_token_to_credentials(config) + # save the config + source.write_config(migrated_config, config_path) + return migrated_config + + @classmethod + def _emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + # add the Airbyte Control Message to message repo + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + # emit the Airbyte Control Message from message queue to stdout + for message in cls.message_repository._message_queue: + print(message.json(exclude_unset=True)) + + @classmethod + def migrate(cls, args: List[str], source: SourceSlack) -> None: + """ + This method checks the input args, should the config be migrated, + transform if necessary and emit the CONTROL message. + """ + # get config path + config_path = AirbyteEntrypoint(source).extract_config(args) + # proceed only if `--config` arg is provided + if config_path: + # read the existing config + config = source.read_config(config_path) + # migration check + if cls._should_migrate(config): + cls._emit_control_message( + cls._modify_and_save(config_path, source, config), + ) diff --git a/airbyte-integrations/connectors/source-slack/source_slack/manifest.yaml b/airbyte-integrations/connectors/source-slack/source_slack/manifest.yaml new file mode 100644 index 000000000000..5a00f9a41ea1 --- /dev/null +++ b/airbyte-integrations/connectors/source-slack/source_slack/manifest.yaml @@ -0,0 +1,260 @@ +version: 0.73.0 +type: DeclarativeSource + +definitions: + schema_loader: + type: JsonFileSchemaLoader + file_path: "./source_slack/schemas/{{ parameters['name'] }}.json" + + default_paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: cursor + page_size_option: + type: RequestOption + field_name: limit + inject_into: request_parameter + pagination_strategy: + type: CursorPagination + page_size: 1000 + cursor_value: '{{ response.get("response_metadata", {}).get("next_cursor", {}) }}' + stop_condition: >- + {{ not response.get("response_metadata", {}).get("next_cursor", {}) + }} + + api_token_auth: + type: BearerAuthenticator + api_token: "{{ config['credentials']['api_token'] }}" + access_token_auth: + type: BearerAuthenticator + api_token: "{{ config['credentials']['access_token'] }}" + + requester: + type: HttpRequester + url_base: https://slack.com/api/ + path: "{{ parameters['path'] }}" + http_method: GET + request_parameters: {} + request_headers: {} + authenticator: + type: SelectiveAuthenticator + authenticator_selection_path: ["credentials", "option_title"] + authenticators: + Default OAuth2.0 authorization: "#/definitions/access_token_auth" + API Token Credentials: "#/definitions/api_token_auth" + request_body_json: {} + error_handler: + type: DefaultErrorHandler + response_filters: + - error_message_contains: "invalid_auth" + action: FAIL + error_message: Authentication has failed, please update your credentials. + - http_codes: [429] + action: RETRY + error_message: Failed to perform a request due to rate limits. + - http_codes: [500, 503] + action: RETRY + error_message: Failed to perform a request due to internal server error. + + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - "{{ parameters['field_path'] }}" + + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/default_paginator" + partition_router: [] + + stream_base: + primary_key: "id" + retriever: + $ref: "#/definitions/retriever" + schema_loader: + $ref: "#/definitions/schema_loader" + + users_stream: + primary_key: "id" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + error_handler: + type: DefaultErrorHandler + response_filters: + - error_message_contains: "invalid_auth" + action: FAIL + error_message: Authentication has failed, please update your credentials. + - http_codes: [429] + action: RETRY + error_message: Failed to perform a request due to rate limits. + - http_codes: [403, 400] + action: FAIL + error_message: Got an exception while trying to set up the connection. Most probably, there are no users in the given Slack instance or your token is incorrect. + - http_codes: [500, 503] + action: RETRY + error_message: Failed to perform a request due to internal server error. + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/default_paginator" + partition_router: [] + schema_loader: + $ref: "#/definitions/schema_loader" + $parameters: + name: users + path: users.list + field_path: members + + channels_stream: + primary_key: "id" + $parameters: + name: channels + path: conversations.list + field_path: channels + schema_loader: + $ref: "#/definitions/schema_loader" + retriever: + class_name: "source_slack.components.join_channels.ChannelsRetriever" + requester: + $ref: "#/definitions/requester" + request_parameters: + types: "public_channel" + record_selector: + $ref: "#/definitions/selector" + record_filter: + type: RecordFilter + condition: "{{ record.name in config.channel_filter or not config.channel_filter }}" + $parameters: + transformations: [[]] + paginator: + $ref: "#/definitions/default_paginator" + $parameters: + url_base: https://slack.com/api/ + partition_router: [] + + channels_partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + request_option: + type: RequestOption + field_name: channel + inject_into: request_parameter + partition_field: channel_id + stream: "#/definitions/channels_stream" + + channel_members_stream: + $ref: "#/definitions/stream_base" + $parameters: + name: channel_members + path: conversations.members + field_path: members + primary_key: + - member_id + - channel_id + retriever: + $ref: "#/definitions/retriever" + partition_router: + $ref: "#/definitions/channels_partition_router" + record_selector: + type: RecordSelector + extractor: + class_name: "source_slack.components.channel_members_extractor.ChannelMembersExtractor" + field_path: ["members"] + transformations: + - type: AddFields + fields: + - path: + - channel_id + value: "{{ stream_partition.get('channel_id') }}" + + channel_messages_stream: + $ref: "#/definitions/stream_base" + $parameters: + name: channel_messages + path: conversations.history + field_path: messages + primary_key: + - channel_id + - ts + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + request_parameters: + inclusive: "True" + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/default_paginator" + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + stream: + $ref: "#/definitions/channels_stream" + $parameters: + name: channels + path: conversations.list + field_path: channels + parent_key: id + partition_field: channel + request_option: + field_name: "channel" + inject_into: "request_parameter" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: float_ts + cursor_datetime_formats: + - "%s" + step: P100D + cursor_granularity: P10D + lookback_window: "P{{ config.get('lookback_window', 0) }}D" + datetime_format: "%s" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + inject_into: request_parameter + field_name: oldest + type: RequestOption + end_time_option: + inject_into: request_parameter + field_name: latest + type: RequestOption + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + transformations: + - type: AddFields + fields: + - path: + - float_ts + value: "{{ record.ts|float }}" + - path: + - channel_id + value: "{{ stream_partition.get('channel') }}" + +streams: + - "#/definitions/users_stream" + - "#/definitions/channels_stream" + - "#/definitions/channel_members_stream" + - "#/definitions/channel_messages_stream" + +check: + type: CheckStream + stream_names: + - users diff --git a/airbyte-integrations/connectors/source-slack/source_slack/run.py b/airbyte-integrations/connectors/source-slack/source_slack/run.py index 14caa9ab08e1..fd5e385857b9 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/run.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/run.py @@ -7,8 +7,10 @@ from airbyte_cdk.entrypoint import launch from source_slack import SourceSlack +from source_slack.config_migrations import MigrateLegacyConfig def run(): source = SourceSlack() + MigrateLegacyConfig.migrate(sys.argv[1:], source) launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-slack/source_slack/source.py b/airbyte-integrations/connectors/source-slack/source_slack/source.py index e785114f865f..3925e4bd44a6 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/source.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/source.py @@ -2,354 +2,21 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -from abc import ABC, abstractmethod -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from typing import Any, List, Mapping import pendulum -import requests -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream +from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from pendulum import DateTime - -from .utils import chunk_date_range - - -class SlackStream(HttpStream, ABC): - url_base = "https://slack.com/api/" - primary_key = "id" - page_size = 1000 - - @property - def max_retries(self) -> int: - # Slack's rate limiting can be unpredictable so we increase the max number of retries by a lot before failing - return 20 - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """Slack uses a cursor-based pagination strategy. - Extract the cursor from the response if it exists and return it in a format - that can be used to update request parameters""" - - json_response = response.json() - next_cursor = json_response.get("response_metadata", {}).get("next_cursor") - if next_cursor: - return {"cursor": next_cursor} - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - params = {"limit": self.page_size} - if next_page_token: - params.update(**next_page_token) - return params - - def parse_response( - self, - response: requests.Response, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[MutableMapping]: - json_response = response.json() - yield from json_response.get(self.data_field, []) - - def backoff_time(self, response: requests.Response) -> Optional[float]: - """This method is called if we run into the rate limit. - Slack puts the retry time in the `Retry-After` response header so we - we return that value. If the response is anything other than a 429 (e.g: 5XX) - fall back on default retry behavior. - Rate Limits Docs: https://api.slack.com/docs/rate-limits#web""" - - if "Retry-After" in response.headers: - return int(response.headers["Retry-After"]) - else: - self.logger.info("Retry-after header not found. Using default backoff value") - return 5 - - @property - @abstractmethod - def data_field(self) -> str: - """The name of the field in the response which contains the data""" - - def should_retry(self, response: requests.Response) -> bool: - return response.status_code == requests.codes.REQUEST_TIMEOUT or super().should_retry(response) - - -class JoinChannelsStream(HttpStream): - """ - This class is a special stream which joins channels because the Slack API only returns messages from channels this bot is in. - Its responses should only be logged for debugging reasons, not read as records. - """ - - url_base = "https://slack.com/api/" - http_method = "POST" - primary_key = "id" - - def __init__(self, channel_filter: List[str] = None, **kwargs): - self.channel_filter = channel_filter or [] - super().__init__(**kwargs) - - def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable: - """ - Override to simply indicate that the specific channel was joined successfully. - This method should not return any data, but should return an empty iterable. - """ - self.logger.info(f"Successfully joined channel: {stream_slice['channel_name']}") - return [] - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - The pagination is not applicable to this Service Stream. - """ - return None - - def path(self, **kwargs) -> str: - return "conversations.join" - - def request_body_json(self, stream_slice: Mapping = None, **kwargs) -> Optional[Mapping]: - return {"channel": stream_slice["channel"]} - - -class ChanneledStream(SlackStream, ABC): - """Slack stream with channel filter""" - - def __init__(self, channel_filter: List[str] = [], join_channels: bool = False, **kwargs): - self.channel_filter = channel_filter - self.join_channels = join_channels - self.kwargs = kwargs - super().__init__(**kwargs) - - @property - def join_channels_stream(self) -> JoinChannelsStream: - return JoinChannelsStream(authenticator=self.kwargs.get("authenticator"), channel_filter=self.channel_filter) - - def should_join_to_channel(self, channel: Mapping[str, Any]) -> bool: - """ - The `is_member` property indicates whether or not the API Bot is already assigned / joined to the channel. - https://api.slack.com/types/conversation#booleans - """ - return self.join_channels and not channel.get("is_member") - - def make_join_channel_slice(self, channel: Mapping[str, Any]) -> Mapping[str, Any]: - channel_id: str = channel.get("id") - channel_name: str = channel.get("name") - self.logger.info(f"Joining Slack Channel: `{channel_name}`") - return {"channel": channel_id, "channel_name": channel_name} - - -class Channels(ChanneledStream): - data_field = "channels" - - @property - def use_cache(self) -> bool: - return True - - def path(self, **kwargs) -> str: - return "conversations.list" - - def request_params(self, **kwargs) -> MutableMapping[str, Any]: - params = super().request_params(**kwargs) - params["types"] = "public_channel" - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[MutableMapping]: - json_response = response.json() - channels = json_response.get(self.data_field, []) - if self.channel_filter: - channels = [channel for channel in channels if channel["name"] in self.channel_filter] - yield from channels - - def read_records(self, sync_mode: SyncMode, **kwargs) -> Iterable[Mapping[str, Any]]: - """ - Override the default `read_records` method to provide the `JoinChannelsStream` functionality, - and be able to read all the channels, not just the ones that already has the API Bot joined. - """ - for channel in super().read_records(sync_mode=sync_mode): - # check the channel should be joined before reading - if self.should_join_to_channel(channel): - # join the channel before reading it - yield from self.join_channels_stream.read_records( - sync_mode=sync_mode, - stream_slice=self.make_join_channel_slice(channel), - ) - # reading the channel data - self.logger.info(f"Reading the channel: `{channel.get('name')}`") - yield channel - - -class ChannelMembers(ChanneledStream): - data_field = "members" - primary_key = ["member_id", "channel_id"] - - def path(self, **kwargs) -> str: - return "conversations.members" - - def request_params(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, **kwargs) - params["channel"] = stream_slice["channel_id"] - return params +from source_slack.streams import Threads - def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping]: - for member_id in super().parse_response(response, **kwargs): - # Slack just returns raw IDs as a string, so we want to put them in a "join table" format - yield {"member_id": member_id, "channel_id": stream_slice["channel_id"]} - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: - channels_stream = Channels(authenticator=self._session.auth, channel_filter=self.channel_filter) - for channel_record in channels_stream.read_records(sync_mode=SyncMode.full_refresh): - yield {"channel_id": channel_record["id"]} +class SourceSlack(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) - -class Users(SlackStream): - data_field = "members" - - def path(self, **kwargs) -> str: - return "users.list" - - -# Incremental Streams -class IncrementalMessageStream(ChanneledStream, ABC): - data_field = "messages" - cursor_field = "float_ts" - primary_key = ["channel_id", "ts"] - - def __init__(self, default_start_date: DateTime, end_date: Optional[DateTime] = None, **kwargs): - self._start_ts = default_start_date.timestamp() - self._end_ts = end_date and end_date.timestamp() - self.set_sub_primary_key() - super().__init__(**kwargs) - - def set_sub_primary_key(self): - if isinstance(self.primary_key, list): - for index, value in enumerate(self.primary_key): - setattr(self, f"sub_primary_key_{index + 1}", value) - else: - self.logger.error("Failed during setting sub primary keys. Primary key should be list.") - - def request_params(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, **kwargs) - params.update(**stream_slice) - return params - - def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping]: - for record in super().parse_response(response, **kwargs): - record[self.sub_primary_key_1] = stream_slice.get("channel", "") - record[self.cursor_field] = float(record[self.sub_primary_key_2]) - yield record - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - current_stream_state = current_stream_state or {} - current_stream_state[self.cursor_field] = max( - latest_record[self.cursor_field], current_stream_state.get(self.cursor_field, self._start_ts) - ) - - return current_stream_state - - def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: - if not stream_slice: - # return an empty iterator - # this is done to emit at least one state message when no slices are generated - return iter([]) - return super().read_records(sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state) - - -class ChannelMessages(HttpSubStream, IncrementalMessageStream): - def path(self, **kwargs) -> str: - return "conversations.history" - - @property - def use_cache(self) -> bool: - return True - - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: - stream_state = stream_state or {} - start_date = pendulum.from_timestamp(stream_state.get(self.cursor_field, self._start_ts)) - end_date = self._end_ts and pendulum.from_timestamp(self._end_ts) - slice_yielded = False - for parent_slice in super().stream_slices(sync_mode=SyncMode.full_refresh): - channel = parent_slice["parent"] - for period in chunk_date_range(start_date=start_date, end_date=end_date): - yield {"channel": channel["id"], "oldest": period.start.timestamp(), "latest": period.end.timestamp()} - slice_yielded = True - if not slice_yielded: - # yield an empty slice to checkpoint state later - yield {} - - -class Threads(IncrementalMessageStream): - def __init__(self, lookback_window: Mapping[str, int], **kwargs): - self.messages_lookback_window = lookback_window - super().__init__(**kwargs) - - def path(self, **kwargs) -> str: - return "conversations.replies" - - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: - """ - The logic for incrementally syncing threads is not very obvious, so buckle up. - - To get all messages in a thread, one must specify the channel and timestamp of the parent (first) message of that thread, - basically its ID. - - One complication is that threads can be updated at Any time in the future. Therefore, if we wanted to comprehensively sync data - i.e: get every single response in a thread, we'd have to read every message in the slack instance every time we ran a sync, - because otherwise there is no way to guarantee that a thread deep in the past didn't receive a new message. - - A pragmatic workaround is to say we want threads to be at least N days fresh i.e: look back N days into the past, - get every message since, and read all of the thread responses. This is essentially the approach we're taking here via slicing: - create slices from N days into the past and read all messages in threads since then. We could optionally filter out records we have - already read, but that's omitted to keep the logic simple to reason about. - - Good luck. - """ - - stream_state = stream_state or {} - channels_stream = Channels(authenticator=self._session.auth, channel_filter=self.channel_filter) - - if self.cursor_field in stream_state: - # Since new messages can be posted to threads continuously after the parent message has been posted, - # we get messages from the latest date - # found in the state minus X days to pick up any new messages in threads. - # If there is state always use lookback - messages_start_date = pendulum.from_timestamp(stream_state[self.cursor_field]) - self.messages_lookback_window - else: - # If there is no state i.e: this is the first sync then there is no use for lookback, just get messages - # from the default start date - messages_start_date = pendulum.from_timestamp(self._start_ts) - - messages_stream = ChannelMessages( - parent=channels_stream, - authenticator=self._session.auth, - default_start_date=messages_start_date, - end_date=self._end_ts and pendulum.from_timestamp(self._end_ts), - ) - - slice_yielded = False - for message_chunk in messages_stream.stream_slices(stream_state={self.cursor_field: messages_start_date.timestamp()}): - self.logger.info(f"Syncing replies {message_chunk}") - for message in messages_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=message_chunk): - yield {"channel": message_chunk["channel"], self.sub_primary_key_2: message[self.sub_primary_key_2]} - slice_yielded = True - if not slice_yielded: - # yield an empty slice to checkpoint state later - yield {} - - -class SourceSlack(AbstractSource): - def _get_authenticator(self, config: Mapping[str, Any]): + def _threads_authenticator(self, config: Mapping[str, Any]): # Added to maintain backward compatibility with previous versions if "api_token" in config: return TokenAuthenticator(config["api_token"]) @@ -363,48 +30,27 @@ def _get_authenticator(self, config: Mapping[str, Any]): else: raise Exception(f"No supported option_title: {credentials_title} specified. See spec.json for references") - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: - try: - authenticator = self._get_authenticator(config) - users_stream = Users(authenticator=authenticator) - next(users_stream.read_records(SyncMode.full_refresh)) - return True, None - except Exception as e: - return ( - False, - f"Got an exception while trying to set up the connection: {e}. " - f"Most probably, there are no users in the given Slack instance or your token is incorrect", - ) - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - authenticator = self._get_authenticator(config) + def get_threads_stream(self, config: Mapping[str, Any]) -> HttpStream: + authenticator = self._threads_authenticator(config) default_start_date = pendulum.parse(config["start_date"]) # this field is not exposed to spec, used only for testing purposes end_date = config.get("end_date") end_date = end_date and pendulum.parse(end_date) threads_lookback_window = pendulum.Duration(days=config["lookback_window"]) channel_filter = config.get("channel_filter", []) - should_join_to_channels = config.get("join_channels") + threads = Threads( + authenticator=authenticator, + default_start_date=default_start_date, + end_date=end_date, + lookback_window=threads_lookback_window, + channel_filter=channel_filter, + ) + return threads + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + declarative_streams = super().streams(config) - channels = Channels(authenticator=authenticator, join_channels=should_join_to_channels, channel_filter=channel_filter) - streams = [ - channels, - ChannelMembers(authenticator=authenticator, channel_filter=channel_filter), - ChannelMessages( - parent=channels, - authenticator=authenticator, - default_start_date=default_start_date, - end_date=end_date, - channel_filter=channel_filter, - ), - Threads( - authenticator=authenticator, - default_start_date=default_start_date, - end_date=end_date, - lookback_window=threads_lookback_window, - channel_filter=channel_filter, - ), - Users(authenticator=authenticator), - ] + threads_stream = self.get_threads_stream(config) + declarative_streams.append(threads_stream) - return streams + return declarative_streams diff --git a/airbyte-integrations/connectors/source-slack/source_slack/spec.json b/airbyte-integrations/connectors/source-slack/source_slack/spec.json index 2ed0ba91abbd..e59f508e41ab 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/spec.json +++ b/airbyte-integrations/connectors/source-slack/source_slack/spec.json @@ -1,5 +1,4 @@ { - "documentationUrl": "https://docs.airbyte.com/integrations/sources/slack", "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Slack Spec", @@ -107,6 +106,7 @@ "predicate_key": ["credentials", "option_title"], "predicate_value": "Default OAuth2.0 authorization", "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": null, "complete_oauth_output_specification": { "type": "object", "additionalProperties": false, diff --git a/airbyte-integrations/connectors/source-slack/source_slack/streams.py b/airbyte-integrations/connectors/source-slack/source_slack/streams.py new file mode 100644 index 000000000000..b565d9670e25 --- /dev/null +++ b/airbyte-integrations/connectors/source-slack/source_slack/streams.py @@ -0,0 +1,280 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC, abstractmethod +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional + +import pendulum +import requests +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream +from pendulum import DateTime + +from .components.join_channels import JoinChannelsStream +from .utils import chunk_date_range + + +class SlackStream(HttpStream, ABC): + url_base = "https://slack.com/api/" + primary_key = "id" + page_size = 1000 + + @property + def max_retries(self) -> int: + # Slack's rate limiting can be unpredictable so we increase the max number of retries by a lot before failing + return 20 + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """Slack uses a cursor-based pagination strategy. + Extract the cursor from the response if it exists and return it in a format + that can be used to update request parameters""" + + json_response = response.json() + next_cursor = json_response.get("response_metadata", {}).get("next_cursor") + if next_cursor: + return {"cursor": next_cursor} + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + params = {"limit": self.page_size} + if next_page_token: + params.update(**next_page_token) + return params + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[MutableMapping]: + json_response = response.json() + yield from json_response.get(self.data_field, []) + + def backoff_time(self, response: requests.Response) -> Optional[float]: + """This method is called if we run into the rate limit. + Slack puts the retry time in the `Retry-After` response header so we + we return that value. If the response is anything other than a 429 (e.g: 5XX) + fall back on default retry behavior. + Rate Limits Docs: https://api.slack.com/docs/rate-limits#web""" + + if "Retry-After" in response.headers: + return int(response.headers["Retry-After"]) + else: + self.logger.info("Retry-after header not found. Using default backoff value") + return 5 + + @property + @abstractmethod + def data_field(self) -> str: + """The name of the field in the response which contains the data""" + + def should_retry(self, response: requests.Response) -> bool: + return response.status_code == requests.codes.REQUEST_TIMEOUT or super().should_retry(response) + + +class ChanneledStream(SlackStream, ABC): + """Slack stream with channel filter""" + + def __init__(self, channel_filter: List[str] = [], join_channels: bool = False, **kwargs): + self.channel_filter = channel_filter + self.join_channels = join_channels + self.kwargs = kwargs + super().__init__(**kwargs) + + @property + def join_channels_stream(self) -> JoinChannelsStream: + return JoinChannelsStream(authenticator=self.kwargs.get("authenticator"), channel_filter=self.channel_filter) + + def should_join_to_channel(self, channel: Mapping[str, Any]) -> bool: + """ + The `is_member` property indicates whether or not the API Bot is already assigned / joined to the channel. + https://api.slack.com/types/conversation#booleans + """ + return self.join_channels and not channel.get("is_member") + + def make_join_channel_slice(self, channel: Mapping[str, Any]) -> Mapping[str, Any]: + channel_id: str = channel.get("id") + channel_name: str = channel.get("name") + self.logger.info(f"Joining Slack Channel: `{channel_name}`") + return {"channel": channel_id, "channel_name": channel_name} + + +class Channels(ChanneledStream): + data_field = "channels" + + @property + def use_cache(self) -> bool: + return True + + def path(self, **kwargs) -> str: + return "conversations.list" + + def request_params(self, **kwargs) -> MutableMapping[str, Any]: + params = super().request_params(**kwargs) + params["types"] = "public_channel" + return params + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[MutableMapping]: + json_response = response.json() + channels = json_response.get(self.data_field, []) + if self.channel_filter: + channels = [channel for channel in channels if channel["name"] in self.channel_filter] + yield from channels + + def read_records(self, sync_mode: SyncMode, **kwargs) -> Iterable[Mapping[str, Any]]: + """ + Override the default `read_records` method to provide the `JoinChannelsStream` functionality, + and be able to read all the channels, not just the ones that already has the API Bot joined. + """ + for channel in super().read_records(sync_mode=sync_mode): + # check the channel should be joined before reading + if self.should_join_to_channel(channel): + # join the channel before reading it + yield from self.join_channels_stream.read_records( + sync_mode=sync_mode, + stream_slice=self.make_join_channel_slice(channel), + ) + # reading the channel data + self.logger.info(f"Reading the channel: `{channel.get('name')}`") + yield channel + + +# Incremental Streams +class IncrementalMessageStream(ChanneledStream, ABC): + data_field = "messages" + cursor_field = "float_ts" + primary_key = ["channel_id", "ts"] + + def __init__(self, default_start_date: DateTime, end_date: Optional[DateTime] = None, **kwargs): + self._start_ts = default_start_date.timestamp() + self._end_ts = end_date and end_date.timestamp() + self.set_sub_primary_key() + super().__init__(**kwargs) + + def set_sub_primary_key(self): + if isinstance(self.primary_key, list): + for index, value in enumerate(self.primary_key): + setattr(self, f"sub_primary_key_{index + 1}", value) + else: + self.logger.error("Failed during setting sub primary keys. Primary key should be list.") + + def request_params(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, **kwargs) + params.update(**stream_slice) + return params + + def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping]: + for record in super().parse_response(response, **kwargs): + record[self.sub_primary_key_1] = stream_slice.get("channel", "") + record[self.cursor_field] = float(record[self.sub_primary_key_2]) + yield record + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + current_stream_state = current_stream_state or {} + current_stream_state[self.cursor_field] = max( + latest_record[self.cursor_field], current_stream_state.get(self.cursor_field, self._start_ts) + ) + + return current_stream_state + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Mapping[str, Any]]: + if not stream_slice: + # return an empty iterator + # this is done to emit at least one state message when no slices are generated + return iter([]) + return super().read_records(sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state) + + +class ChannelMessages(HttpSubStream, IncrementalMessageStream): + def path(self, **kwargs) -> str: + return "conversations.history" + + @property + def use_cache(self) -> bool: + return True + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + stream_state = stream_state or {} + start_date = pendulum.from_timestamp(stream_state.get(self.cursor_field, self._start_ts)) + end_date = self._end_ts and pendulum.from_timestamp(self._end_ts) + slice_yielded = False + for parent_slice in super().stream_slices(sync_mode=SyncMode.full_refresh): + channel = parent_slice["parent"] + for period in chunk_date_range(start_date=start_date, end_date=end_date): + yield {"channel": channel["id"], "oldest": period.start.timestamp(), "latest": period.end.timestamp()} + slice_yielded = True + if not slice_yielded: + # yield an empty slice to checkpoint state later + yield {} + + +class Threads(IncrementalMessageStream): + def __init__(self, lookback_window: Mapping[str, int], **kwargs): + self.messages_lookback_window = lookback_window + super().__init__(**kwargs) + + def path(self, **kwargs) -> str: + return "conversations.replies" + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + """ + The logic for incrementally syncing threads is not very obvious, so buckle up. + + To get all messages in a thread, one must specify the channel and timestamp of the parent (first) message of that thread, + basically its ID. + + One complication is that threads can be updated at Any time in the future. Therefore, if we wanted to comprehensively sync data + i.e: get every single response in a thread, we'd have to read every message in the slack instance every time we ran a sync, + because otherwise there is no way to guarantee that a thread deep in the past didn't receive a new message. + + A pragmatic workaround is to say we want threads to be at least N days fresh i.e: look back N days into the past, + get every message since, and read all of the thread responses. This is essentially the approach we're taking here via slicing: + create slices from N days into the past and read all messages in threads since then. We could optionally filter out records we have + already read, but that's omitted to keep the logic simple to reason about. + + Good luck. + """ + + stream_state = stream_state or {} + channels_stream = Channels(authenticator=self._session.auth, channel_filter=self.channel_filter) + + if self.cursor_field in stream_state: + # Since new messages can be posted to threads continuously after the parent message has been posted, + # we get messages from the latest date + # found in the state minus X days to pick up any new messages in threads. + # If there is state always use lookback + messages_start_date = pendulum.from_timestamp(stream_state[self.cursor_field]) - self.messages_lookback_window + else: + # If there is no state i.e: this is the first sync then there is no use for lookback, just get messages + # from the default start date + messages_start_date = pendulum.from_timestamp(self._start_ts) + + messages_stream = ChannelMessages( + parent=channels_stream, + authenticator=self._session.auth, + default_start_date=messages_start_date, + end_date=self._end_ts and pendulum.from_timestamp(self._end_ts), + ) + + slice_yielded = False + for message_chunk in messages_stream.stream_slices(stream_state={self.cursor_field: messages_start_date.timestamp()}): + self.logger.info(f"Syncing replies {message_chunk}") + for message in messages_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=message_chunk): + yield {"channel": message_chunk["channel"], self.sub_primary_key_2: message[self.sub_primary_key_2]} + slice_yielded = True + if not slice_yielded: + # yield an empty slice to checkpoint state later + yield {} diff --git a/airbyte-integrations/connectors/source-slack/source_slack/utils.py b/airbyte-integrations/connectors/source-slack/source_slack/utils.py index 7507dbab3565..febfb788442a 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/utils.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/utils.py @@ -9,7 +9,7 @@ from pendulum import DateTime, Period -def chunk_date_range(start_date: DateTime, interval=pendulum.duration(days=1), end_date: Optional[DateTime] = None) -> Iterable[Period]: +def chunk_date_range(start_date: DateTime, interval=pendulum.duration(days=100), end_date: Optional[DateTime] = None) -> Iterable[Period]: """ Yields a list of the beginning and ending timestamps of each day between the start date and now. The return value is a pendulum.period diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/configs/actual_config.json b/airbyte-integrations/connectors/source-slack/unit_tests/configs/actual_config.json new file mode 100644 index 000000000000..065ffde78394 --- /dev/null +++ b/airbyte-integrations/connectors/source-slack/unit_tests/configs/actual_config.json @@ -0,0 +1,11 @@ +{ + "start_date": "2021-07-22T20:00:00Z", + "end_date": "2021-07-23T20:00:00Z", + "lookback_window": 1, + "join_channels": true, + "channel_filter": ["airbyte-for-beginners", "good-reads"], + "credentials": { + "api_token": "api-token", + "option_title": "API Token Credentials" + } +} diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/configs/legacy_config.json b/airbyte-integrations/connectors/source-slack/unit_tests/configs/legacy_config.json new file mode 100644 index 000000000000..99eda1d750b3 --- /dev/null +++ b/airbyte-integrations/connectors/source-slack/unit_tests/configs/legacy_config.json @@ -0,0 +1,8 @@ +{ + "start_date": "2021-07-22T20:00:00Z", + "end_date": "2021-07-23T20:00:00Z", + "lookback_window": 1, + "join_channels": true, + "channel_filter": ["airbyte-for-beginners", "good-reads"], + "api_token": "api-token" +} diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py b/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py index 6d9254730d5f..002a9ec96779 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py @@ -18,20 +18,11 @@ def conversations_list(requests_mock): "https://slack.com/api/conversations.list?limit=1000&types=public_channel", json={ "channels": [ - {"name": "advice-data-architecture", "id": 1, "is_member": False}, - {"name": "advice-data-orchestration", "id": 2, "is_member": True}, - {"name": "airbyte-for-beginners", "id": 3, "is_member": False}, - {"name": "good-reads", "id": 4, "is_member": True}, - ] + {"id": "airbyte-for-beginners", "is_member": True}, + {"id": "good-reads", "is_member": True}] }, ) - -@pytest.fixture(autouse=True) -def join_channels(requests_mock): - return requests_mock.register_uri("POST", "https://slack.com/api/conversations.join") - - def base_config() -> MutableMapping: return copy.deepcopy( { @@ -100,7 +91,18 @@ def invalid_config() -> MutableMapping: ( (_token_config(), True), (_oauth_config(), True), - (_legacy_token_config(), True), (_invalid_config(), False), ), ) + + +@pytest.fixture +def joined_channel(): + return {"id": "C061EG9SL", "name": "general", "is_channel": True, "is_group": False, "is_im": False, + "created": 1449252889, + "creator": "U061F7AUR", "is_archived": False, "is_general": True, "unlinked": 0, "name_normalized": "general", + "is_shared": False, + "is_ext_shared": False, "is_org_shared": False, "pending_shared": [], "is_pending_ext_shared": False, + "is_member": True, "is_private": False, "is_mpim": False, + "topic": {"value": "Which widget do you worry about?", "creator": "", "last_set": 0}, + "purpose": {"value": "For widget discussion", "creator": "", "last_set": 0}, "previous_names": []} diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/test_components.py b/airbyte-integrations/connectors/source-slack/unit_tests/test_components.py new file mode 100644 index 000000000000..e40e700fe97b --- /dev/null +++ b/airbyte-integrations/connectors/source-slack/unit_tests/test_components.py @@ -0,0 +1,99 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from unittest.mock import MagicMock + +import pendulum +import pytest +from airbyte_cdk.sources.declarative.extractors import DpathExtractor, RecordSelector +from airbyte_cdk.sources.declarative.requesters import HttpRequester +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from airbyte_protocol.models import SyncMode +from source_slack import SourceSlack +from source_slack.components.channel_members_extractor import ChannelMembersExtractor +from source_slack.components.join_channels import ChannelsRetriever, JoinChannelsStream + + +def get_stream_by_name(stream_name, config): + streams = SourceSlack().streams(config=config) + for stream in streams: + if stream.name == stream_name: + return stream + raise ValueError(f"Stream {stream_name} not found") + + +def test_channel_members_extractor(token_config): + response_mock = MagicMock() + response_mock.json.return_value = {"members": [ + "U023BECGF", + "U061F7AUR", + "W012A3CDE" + ]} + records = ChannelMembersExtractor(config=token_config, parameters={}, field_path=["members"]).extract_records(response=response_mock) + assert records == [{"member_id": "U023BECGF"}, + {"member_id": "U061F7AUR"}, + {"member_id": "W012A3CDE"}] + + +def test_join_channels(token_config, requests_mock, joined_channel): + mocked_request = requests_mock.post( + url="https://slack.com/api/conversations.join", + json={"ok": True, "channel": joined_channel} + ) + token = token_config["credentials"]["api_token"] + authenticator = TokenAuthenticator(token) + channel_filter = token_config["channel_filter"] + stream = JoinChannelsStream(authenticator=authenticator, channel_filter=channel_filter) + records = stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"channel": "C061EG9SL", "channel_name": "general"}) + assert not list(records) + assert mocked_request.called + + +def get_channels_retriever_instance(token_config): + return ChannelsRetriever( + config=token_config, + requester=HttpRequester(name="channels", path="conversations.list", url_base="https://slack.com/api/", config=token_config, + parameters={}), + record_selector=RecordSelector( + extractor=DpathExtractor(field_path=["channels"], config=token_config, parameters={}), + config=token_config, parameters={}, + schema_normalization=None), + parameters={} + ) + + +def test_join_channels_should_join_to_channel(token_config): + retriever = get_channels_retriever_instance(token_config) + assert retriever.should_join_to_channel(token_config, {"is_member": False}) is True + assert retriever.should_join_to_channel(token_config, {"is_member": True}) is False + + +def test_join_channels_make_join_channel_slice(token_config): + retriever = get_channels_retriever_instance(token_config) + expected_slice = {"channel": "C061EG9SL", "channel_name": "general"} + assert retriever.make_join_channel_slice({"id": "C061EG9SL", "name": "general"}) == expected_slice + + +@pytest.mark.parametrize( + "join_response, log_message", + ( + ({"ok": True, "channel": {"is_member": True, "id": "channel 2", "name": "test channel"}}, "Successfully joined channel: test channel"), + ({"ok": False, "error": "missing_scope", "needed": "channels:write"}, + "Unable to joined channel: test channel. Reason: {'ok': False, 'error': " "'missing_scope', 'needed': 'channels:write'}"), + ), + ids=["successful_join_to_channel", "failed_join_to_channel"] +) +def test_join_channel_read(requests_mock, token_config, joined_channel, caplog, join_response, log_message): + mocked_request = requests_mock.post( + url="https://slack.com/api/conversations.join", + json=join_response + ) + requests_mock.get( + url="https://slack.com/api/conversations.list", + json={"channels": [{"is_member": True, "id": "channel 1"}, {"is_member": False, "id": "channel 2", "name": "test channel"}]} + ) + + retriever = get_channels_retriever_instance(token_config) + assert len(list(retriever.read_records(records_schema={}))) == 2 + assert mocked_request.called + assert mocked_request.last_request._request.body == b'{"channel": "channel 2"}' + assert log_message in caplog.text diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-slack/unit_tests/test_config_migrations.py new file mode 100644 index 000000000000..761597a66fc2 --- /dev/null +++ b/airbyte-integrations/connectors/source-slack/unit_tests/test_config_migrations.py @@ -0,0 +1,47 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +import os +from typing import Any, Mapping + +from source_slack import SourceSlack +from source_slack.config_migrations import MigrateLegacyConfig + +CMD = "check" +TEST_CONFIG_LEGACY_PATH = f"{os.path.dirname(__file__)}/configs/legacy_config.json" +TEST_CONFIG_ACTUAL_PATH = f"{os.path.dirname(__file__)}/configs/actual_config.json" + +SOURCE_INPUT_ARGS_LEGACY = [CMD, "--config", TEST_CONFIG_LEGACY_PATH] +SOURCE_INPUT_ARGS_ACTUAL = [CMD, "--config", TEST_CONFIG_ACTUAL_PATH] + + +def revert_config(): + with open(TEST_CONFIG_LEGACY_PATH, "r") as test_config: + config = json.load(test_config) + config.pop("credentials") + config.update({"api_token": "api-token"}) + with open(TEST_CONFIG_LEGACY_PATH, "w") as updated_config: + config = json.dumps(config) + updated_config.write(config) + + +def load_config(config_path: str = TEST_CONFIG_LEGACY_PATH) -> Mapping[str, Any]: + with open(config_path, "r") as config: + return json.load(config) + + +def test_config_migration(): + migration = MigrateLegacyConfig() + migration.migrate(SOURCE_INPUT_ARGS_LEGACY, SourceSlack()) + test_migrated_config = load_config() + assert test_migrated_config["credentials"]["api_token"] == "api-token" + assert test_migrated_config["credentials"]["option_title"] == "API Token Credentials" + revert_config() + + +def test_config_not_migrated(): + config_before_migration = load_config(TEST_CONFIG_ACTUAL_PATH) + migration = MigrateLegacyConfig() + migration.migrate(SOURCE_INPUT_ARGS_ACTUAL, SourceSlack()) + test_migrated_config = load_config(TEST_CONFIG_ACTUAL_PATH) + assert config_before_migration == test_migrated_config diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py b/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py index bef3bf26651f..ae1a58922797 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/test_source.py @@ -10,6 +10,13 @@ from .conftest import parametrized_configs +def get_stream_by_name(stream_name, config): + streams = SourceSlack().streams(config=config) + for stream in streams: + if stream.name == stream_name: + return stream + raise ValueError(f"Stream {stream_name} not found") + @parametrized_configs def test_streams(conversations_list, config, is_valid): source = SourceSlack() @@ -19,28 +26,25 @@ def test_streams(conversations_list, config, is_valid): else: with pytest.raises(Exception) as exc_info: _ = source.streams(config) - assert "No supported option_title: None specified. See spec.json for references" in repr(exc_info.value) + assert "The path from `authenticator_selection_path` is not found in the config." in repr(exc_info.value) @pytest.mark.parametrize( "status_code, response, is_connection_successful, error_msg", ( (200, {"members": [{"id": 1, "name": "Abraham"}]}, True, None), + (200, {"ok": False, "error": "invalid_auth"}, False, "Authentication has failed, please update your credentials."), ( 400, "Bad request", False, - "Got an exception while trying to set up the connection: 400 Client Error: " - "None for url: https://slack.com/api/users.list?limit=1000. Most probably, there are no users in the given Slack instance or " - "your token is incorrect", + "Got an exception while trying to set up the connection. Most probably, there are no users in the given Slack instance or your token is incorrect.", ), ( 403, "Forbidden", False, - "Got an exception while trying to set up the connection: 403 Client Error: " - "None for url: https://slack.com/api/users.list?limit=1000. Most probably, there are no users in the given Slack instance or " - "your token is incorrect", + "Got an exception while trying to set up the connection. Most probably, there are no users in the given Slack instance or your token is incorrect.", ), ), ) @@ -49,4 +53,20 @@ def test_check_connection(token_config, requests_mock, status_code, response, is source = SourceSlack() success, error = source.check_connection(logger=logging.getLogger("airbyte"), config=token_config) assert success is is_connection_successful - assert error == error_msg + if not success: + assert error_msg in error + + +def test_threads_auth(token_config, oauth_config): + source = SourceSlack() + auth = source._threads_authenticator(token_config) + assert auth.token == "Bearer api-token" + source = SourceSlack() + auth = source._threads_authenticator(oauth_config) + assert auth.token == "Bearer access-token" + + +def test_get_threads_stream(token_config): + source = SourceSlack() + threads_stream = source.get_threads_stream(token_config) + assert threads_stream diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py index d0327093318f..9a3cd092d90b 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/test_streams.py @@ -7,12 +7,21 @@ import pendulum import pytest from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from source_slack.source import Channels, Threads, Users +from source_slack import SourceSlack +from source_slack.streams import Channels, JoinChannelsStream, Threads @pytest.fixture -def authenticator(legacy_token_config): - return TokenAuthenticator(legacy_token_config["api_token"]) +def authenticator(token_config): + return TokenAuthenticator(token_config["credentials"]["api_token"]) + + +def get_stream_by_name(stream_name, config): + streams = SourceSlack().streams(config=config) + for stream in streams: + if stream.name == stream_name: + return stream + raise ValueError(f"Stream {stream_name} not found") @pytest.mark.parametrize( @@ -25,10 +34,10 @@ def authenticator(legacy_token_config): {}, [ # two messages per each channel - {"channel": 3, "ts": 1577866844}, - {"channel": 3, "ts": 1577877406}, - {"channel": 4, "ts": 1577866844}, - {"channel": 4, "ts": 1577877406}, + {'channel': 'airbyte-for-beginners', 'ts': 1577866844}, + {'channel': 'airbyte-for-beginners', 'ts': 1577877406}, + {'channel': 'good-reads', 'ts': 1577866844}, + {'channel': 'good-reads', 'ts': 1577877406}, ], ), ("2020-01-02T00:00:00Z", "2020-01-01T00:00:00Z", [], {}, [{}]), @@ -36,35 +45,36 @@ def authenticator(legacy_token_config): "2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z", [{"ts": 1577866844}, {"ts": 1577877406}], - {"float_ts": 1577915266}, + {"float_ts": 2577866844}, [ - # two messages per each channel per datetime slice - {"channel": 3, "ts": 1577866844}, - {"channel": 3, "ts": 1577877406}, - {"channel": 3, "ts": 1577866844}, - {"channel": 3, "ts": 1577877406}, - {"channel": 4, "ts": 1577866844}, - {"channel": 4, "ts": 1577877406}, - {"channel": 4, "ts": 1577866844}, - {"channel": 4, "ts": 1577877406}, + # no slice when state greater than ts + {}, ], ), ), ) def test_threads_stream_slices( - requests_mock, authenticator, legacy_token_config, start_date, end_date, messages, stream_state, expected_result + requests_mock, authenticator, token_config, start_date, end_date, messages, stream_state, expected_result ): + token_config["channel_filter"] = [] + + requests_mock.register_uri( + "GET", "https://slack.com/api/conversations.history?limit=1000&channel=airbyte-for-beginners", + [{"json": {"messages": messages}}, {"json": {"messages": []}}] + ) requests_mock.register_uri( - "GET", "https://slack.com/api/conversations.history", [{"json": {"messages": messages}}, {"json": {"messages": messages}}] + "GET", "https://slack.com/api/conversations.history?limit=1000&channel=good-reads", + [{"json": {"messages": messages}}, {"json": {"messages": []}}] ) + start_date = pendulum.parse(start_date) end_date = end_date and pendulum.parse(end_date) + stream = Threads( authenticator=authenticator, default_start_date=start_date, end_date=end_date, - lookback_window=pendulum.Duration(days=legacy_token_config["lookback_window"]), - channel_filter=legacy_token_config["channel_filter"], + lookback_window=pendulum.Duration(days=token_config["lookback_window"]) ) slices = list(stream.stream_slices(stream_state=stream_state)) assert slices == expected_result @@ -79,19 +89,64 @@ def test_threads_stream_slices( ({"float_ts": 1577800844}, {"float_ts": 1577866844}, {"float_ts": 1577866844}), ), ) -def test_get_updated_state(authenticator, legacy_token_config, current_state, latest_record, expected_state): +def test_get_updated_state(authenticator, token_config, current_state, latest_record, expected_state): + stream = Threads( authenticator=authenticator, - default_start_date=pendulum.parse(legacy_token_config["start_date"]), - lookback_window=legacy_token_config["lookback_window"], - channel_filter=legacy_token_config["channel_filter"], + default_start_date=pendulum.parse(token_config["start_date"]), + lookback_window=token_config["lookback_window"] ) assert stream.get_updated_state(current_stream_state=current_state, latest_record=latest_record) == expected_state +def test_threads_request_params(authenticator, token_config): + stream = Threads( + authenticator=authenticator, + default_start_date=pendulum.parse(token_config["start_date"]), + lookback_window=token_config["lookback_window"] + ) + threads_slice = {'channel': 'airbyte-for-beginners', 'ts': 1577866844} + expected = {'channel': 'airbyte-for-beginners', 'limit': 1000, 'ts': 1577866844} + assert stream.request_params(stream_slice=threads_slice, stream_state={}) == expected + + +def test_threads_parse_response(mocker, authenticator, token_config): + stream = Threads( + authenticator=authenticator, + default_start_date=pendulum.parse(token_config["start_date"]), + lookback_window=token_config["lookback_window"] + ) + resp = { + "messages": [ + { + "type": "message", + "user": "U061F7AUR", + "text": "island", + "thread_ts": "1482960137.003543", + "reply_count": 3, + "subscribed": True, + "last_read": "1484678597.521003", + "unread_count": 0, + "ts": "1482960137.003543" + } + ] + } + resp_mock = mocker.Mock() + resp_mock.json.return_value = resp + threads_slice = {'channel': 'airbyte-for-beginners', 'ts': 1577866844} + actual_response = list(stream.parse_response(response=resp_mock,stream_slice=threads_slice)) + assert len(actual_response) == 1 + assert actual_response[0]["float_ts"] == 1482960137.003543 + assert actual_response[0]["channel_id"] == "airbyte-for-beginners" + + @pytest.mark.parametrize("headers, expected_result", (({}, 5), ({"Retry-After": 15}, 15))) -def test_backoff(authenticator, headers, expected_result): - stream = Users(authenticator=authenticator) +def test_backoff(token_config, authenticator, headers, expected_result): + stream = Threads( + authenticator=authenticator, + default_start_date=pendulum.parse(token_config["start_date"]), + lookback_window=token_config["lookback_window"] + ) assert stream.backoff_time(Mock(headers=headers)) == expected_result @@ -100,11 +155,38 @@ def test_channels_stream_with_autojoin(authenticator) -> None: The test uses the `conversations_list` fixture(autouse=true) as API mocker. """ expected = [ - {'name': 'advice-data-architecture', 'id': 1, 'is_member': False}, - {'name': 'advice-data-orchestration', 'id': 2, 'is_member': True}, - {'name': 'airbyte-for-beginners', 'id': 3, 'is_member': False}, - {'name': 'good-reads', 'id': 4, 'is_member': True}, + {'id': 'airbyte-for-beginners', 'is_member': True}, + {'id': 'good-reads', 'is_member': True} ] stream = Channels(channel_filter=[], join_channels=True, authenticator=authenticator) assert list(stream.read_records(None)) == expected - \ No newline at end of file + + +def test_next_page_token(authenticator, token_config): + stream = Threads( + authenticator=authenticator, + default_start_date=pendulum.parse(token_config["start_date"]), + lookback_window=token_config["lookback_window"] + ) + mocked_response = Mock() + mocked_response.json.return_value = {"response_metadata": {"next_cursor": "next page"}} + assert stream.next_page_token(mocked_response) == {"cursor": "next page"} + + +@pytest.mark.parametrize( + "status_code, expected", + ( + (200, False), + (403, False), + (429, True), + (500, True), + ), +) +def test_should_retry(authenticator, token_config, status_code, expected): + stream = Threads( + authenticator=authenticator, + default_start_date=pendulum.parse(token_config["start_date"]), + lookback_window=token_config["lookback_window"] + ) + mocked_response = Mock(status_code=status_code) + assert stream.should_retry(mocked_response) == expected diff --git a/docs/integrations/sources/slack-migrations.md b/docs/integrations/sources/slack-migrations.md new file mode 100644 index 000000000000..31458bc54c1c --- /dev/null +++ b/docs/integrations/sources/slack-migrations.md @@ -0,0 +1,18 @@ +# Slack Migration Guide + +## Upgrading to 1.0.0 + +We're continuously striving to enhance the quality and reliability of our connectors at Airbyte. +As part of our commitment to delivering exceptional service, we are transitioning source Slack from the +Python Connector Development Kit (CDK) to our innovative low-code framework. +This is part of a strategic move to streamline many processes across connectors, bolstering maintainability and +freeing us to focus more of our efforts on improving the performance and features of our evolving platform and growing catalog. +However, due to differences between the Python and low-code CDKs, this migration constitutes a breaking change. + +We’ve evolved and standardized how state is managed for incremental streams that are nested within a parent stream. +This change impacts how individual states are tracked and stored for each partition, using a more structured approach +to ensure the most granular and flexible state management. +This change will affect the `Channel Messages` stream. + +## Migration Steps +* A `reset` for `Channel Messages` stream is required after upgrading to this version. diff --git a/docs/integrations/sources/slack.md b/docs/integrations/sources/slack.md index 9df24431ec9c..a8db6c5c6ed2 100644 --- a/docs/integrations/sources/slack.md +++ b/docs/integrations/sources/slack.md @@ -161,40 +161,41 @@ Slack has [rate limit restrictions](https://api.slack.com/docs/rate-limits). ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------| +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------| +| 1.0.0 | 2024-04-02 | [35477](https://github.com/airbytehq/airbyte/pull/35477) | Migration to low-code CDK | | 0.4.1 | 2024-03-27 | [36579](https://github.com/airbytehq/airbyte/pull/36579) | Upgrade airbyte-cdk version to emit record counts as floats | -| 0.4.0 | 2024-03-19 | [36267](https://github.com/airbytehq/airbyte/pull/36267) | Pin airbyte-cdk version to `^0` | -| 0.3.9 | 2024-02-12 | [35157](https://github.com/airbytehq/airbyte/pull/35157) | Manage dependencies with Poetry. | +| 0.4.0 | 2024-03-19 | [36267](https://github.com/airbytehq/airbyte/pull/36267) | Pin airbyte-cdk version to `^0` | +| 0.3.9 | 2024-02-12 | [35157](https://github.com/airbytehq/airbyte/pull/35157) | Manage dependencies with Poetry. | | 0.3.8 | 2024-02-09 | [35131](https://github.com/airbytehq/airbyte/pull/35131) | Fixed the issue when `schema discovery` fails with `502` due to the platform timeout | -| 0.3.7 | 2024-01-10 | [1234](https://github.com/airbytehq/airbyte/pull/1234) | prepare for airbyte-lib | -| 0.3.6 | 2023-11-21 | [32707](https://github.com/airbytehq/airbyte/pull/32707) | Threads: do not use client-side record filtering | -| 0.3.5 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | -| 0.3.4 | 2023-10-06 | [31134](https://github.com/airbytehq/airbyte/pull/31134) | Update CDK and remove non iterable return from records | -| 0.3.3 | 2023-09-28 | [30580](https://github.com/airbytehq/airbyte/pull/30580) | Add `bot_id` field to threads schema | -| 0.3.2 | 2023-09-20 | [30613](https://github.com/airbytehq/airbyte/pull/30613) | Set default value for channel_filters during discover | -| 0.3.1 | 2023-09-19 | [30570](https://github.com/airbytehq/airbyte/pull/30570) | Use default availability strategy | -| 0.3.0 | 2023-09-18 | [30521](https://github.com/airbytehq/airbyte/pull/30521) | Add unexpected fields to streams `channel_messages`, `channels`, `threads`, `users` | -| 0.2.0 | 2023-05-24 | [26497](https://github.com/airbytehq/airbyte/pull/26497) | Fixed `lookback window` value limitations | -| 0.1.26 | 2023-05-17 | [26186](https://github.com/airbytehq/airbyte/pull/26186) | Limited the `lookback window` range for input configuration | -| 0.1.25 | 2023-03-20 | [22889](https://github.com/airbytehq/airbyte/pull/22889) | Specified date formatting in specification | -| 0.1.24 | 2023-03-20 | [24126](https://github.com/airbytehq/airbyte/pull/24126) | Increase page size to 1000 | -| 0.1.23 | 2023-02-21 | [21907](https://github.com/airbytehq/airbyte/pull/21907) | Do not join channels that not gonna be synced | -| 0.1.22 | 2023-01-27 | [22022](https://github.com/airbytehq/airbyte/pull/22022) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 0.1.21 | 2023-01-12 | [21321](https://github.com/airbytehq/airbyte/pull/21321) | Retry Timeout error | -| 0.1.20 | 2022-12-21 | [20767](https://github.com/airbytehq/airbyte/pull/20767) | Update schema | -| 0.1.19 | 2022-12-01 | [19970](https://github.com/airbytehq/airbyte/pull/19970) | Remove OAuth2.0 broken `refresh_token` support | -| 0.1.18 | 2022-09-28 | [17315](https://github.com/airbytehq/airbyte/pull/17315) | Always install latest version of Airbyte CDK | -| 0.1.17 | 2022-08-28 | [16085](https://github.com/airbytehq/airbyte/pull/16085) | Increase unit test coverage | -| 0.1.16 | 2022-08-28 | [16050](https://github.com/airbytehq/airbyte/pull/16050) | Fix SATs | -| 0.1.15 | 2022-03-31 | [11613](https://github.com/airbytehq/airbyte/pull/11613) | Add 'channel_filter' config and improve performance | -| 0.1.14 | 2022-01-26 | [9575](https://github.com/airbytehq/airbyte/pull/9575) | Correct schema | -| 0.1.13 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | -| 0.1.12 | 2021-10-07 | [6570](https://github.com/airbytehq/airbyte/pull/6570) | Implement OAuth support with OAuth authenticator | -| 0.1.11 | 2021-08-27 | [5830](https://github.com/airbytehq/airbyte/pull/5830) | Fix sync operations hang forever issue | -| 0.1.10 | 2021-08-27 | [5697](https://github.com/airbytehq/airbyte/pull/5697) | Fix max retries issue | -| 0.1.9 | 2021-07-20 | [4860](https://github.com/airbytehq/airbyte/pull/4860) | Fix reading threads issue | -| 0.1.8 | 2021-07-14 | [4683](https://github.com/airbytehq/airbyte/pull/4683) | Add float\_ts primary key | -| 0.1.7 | 2021-06-25 | [3978](https://github.com/airbytehq/airbyte/pull/3978) | Release Slack CDK Connector | +| 0.3.7 | 2024-01-10 | [1234](https://github.com/airbytehq/airbyte/pull/1234) | prepare for airbyte-lib | +| 0.3.6 | 2023-11-21 | [32707](https://github.com/airbytehq/airbyte/pull/32707) | Threads: do not use client-side record filtering | +| 0.3.5 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.3.4 | 2023-10-06 | [31134](https://github.com/airbytehq/airbyte/pull/31134) | Update CDK and remove non iterable return from records | +| 0.3.3 | 2023-09-28 | [30580](https://github.com/airbytehq/airbyte/pull/30580) | Add `bot_id` field to threads schema | +| 0.3.2 | 2023-09-20 | [30613](https://github.com/airbytehq/airbyte/pull/30613) | Set default value for channel_filters during discover | +| 0.3.1 | 2023-09-19 | [30570](https://github.com/airbytehq/airbyte/pull/30570) | Use default availability strategy | +| 0.3.0 | 2023-09-18 | [30521](https://github.com/airbytehq/airbyte/pull/30521) | Add unexpected fields to streams `channel_messages`, `channels`, `threads`, `users` | +| 0.2.0 | 2023-05-24 | [26497](https://github.com/airbytehq/airbyte/pull/26497) | Fixed `lookback window` value limitations | +| 0.1.26 | 2023-05-17 | [26186](https://github.com/airbytehq/airbyte/pull/26186) | Limited the `lookback window` range for input configuration | +| 0.1.25 | 2023-03-20 | [22889](https://github.com/airbytehq/airbyte/pull/22889) | Specified date formatting in specification | +| 0.1.24 | 2023-03-20 | [24126](https://github.com/airbytehq/airbyte/pull/24126) | Increase page size to 1000 | +| 0.1.23 | 2023-02-21 | [21907](https://github.com/airbytehq/airbyte/pull/21907) | Do not join channels that not gonna be synced | +| 0.1.22 | 2023-01-27 | [22022](https://github.com/airbytehq/airbyte/pull/22022) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.1.21 | 2023-01-12 | [21321](https://github.com/airbytehq/airbyte/pull/21321) | Retry Timeout error | +| 0.1.20 | 2022-12-21 | [20767](https://github.com/airbytehq/airbyte/pull/20767) | Update schema | +| 0.1.19 | 2022-12-01 | [19970](https://github.com/airbytehq/airbyte/pull/19970) | Remove OAuth2.0 broken `refresh_token` support | +| 0.1.18 | 2022-09-28 | [17315](https://github.com/airbytehq/airbyte/pull/17315) | Always install latest version of Airbyte CDK | +| 0.1.17 | 2022-08-28 | [16085](https://github.com/airbytehq/airbyte/pull/16085) | Increase unit test coverage | +| 0.1.16 | 2022-08-28 | [16050](https://github.com/airbytehq/airbyte/pull/16050) | Fix SATs | +| 0.1.15 | 2022-03-31 | [11613](https://github.com/airbytehq/airbyte/pull/11613) | Add 'channel_filter' config and improve performance | +| 0.1.14 | 2022-01-26 | [9575](https://github.com/airbytehq/airbyte/pull/9575) | Correct schema | +| 0.1.13 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | +| 0.1.12 | 2021-10-07 | [6570](https://github.com/airbytehq/airbyte/pull/6570) | Implement OAuth support with OAuth authenticator | +| 0.1.11 | 2021-08-27 | [5830](https://github.com/airbytehq/airbyte/pull/5830) | Fix sync operations hang forever issue | +| 0.1.10 | 2021-08-27 | [5697](https://github.com/airbytehq/airbyte/pull/5697) | Fix max retries issue | +| 0.1.9 | 2021-07-20 | [4860](https://github.com/airbytehq/airbyte/pull/4860) | Fix reading threads issue | +| 0.1.8 | 2021-07-14 | [4683](https://github.com/airbytehq/airbyte/pull/4683) | Add float\_ts primary key | +| 0.1.7 | 2021-06-25 | [3978](https://github.com/airbytehq/airbyte/pull/3978) | Release Slack CDK Connector |