Source Notion: add validation checks to start_date (#31507)

Co-authored-by: ChristoGrab <ChristoGrab@users.noreply.github.com>

airbyte-integrations/connectors/source-notion/Dockerfile

@@ -34,5 +34,5 @@ COPY source_notion ./source_notion
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=2.0.0
+LABEL io.airbyte.version=2.0.1
 LABEL io.airbyte.name=airbyte/source-notion

airbyte-integrations/connectors/source-notion/metadata.yaml

@@ -5,7 +5,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 6e00b415-b02e-4160-bf02-58176a0ae687
-  dockerImageTag: 2.0.0
+  dockerImageTag: 2.0.1
   dockerRepository: airbyte/source-notion
   githubIssueLabel: source-notion
   icon: notion.svg

airbyte-integrations/connectors/source-notion/setup.py

@@ -7,6 +7,7 @@
 
 MAIN_REQUIREMENTS = [
     "airbyte-cdk",
+    "pendulum==2.1.2",
 ]
 
 TEST_REQUIREMENTS = [

airbyte-integrations/connectors/source-notion/source_notion/source.py

@@ -4,14 +4,17 @@
 
 
 import logging
+import re
 from itertools import islice
 from typing import Any, List, Mapping, Tuple
 
+import pendulum
 import requests
 from airbyte_cdk.models import SyncMode
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator
+from pendulum.parsing.exceptions import ParserError
 
 from .streams import Blocks, Comments, Databases, Pages, Users
 
@@ -30,7 +33,29 @@ def _get_authenticator(self, config: Mapping[str, Any]) -> TokenAuthenticator:
         if config.get("access_token"):
             return TokenAuthenticator(config["access_token"])
 
+    def _validate_start_date(self, config: Mapping[str, Any]):
+        start_date = config.get("start_date")
+
+        if start_date:
+            pattern = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z")
+            if not pattern.match(start_date):  # Compare against the pattern descriptor.
+                return "Please check the format of the start date against the pattern descriptor."
+
+            try:  # Handle invalid dates.
+                parsed_start_date = pendulum.parse(start_date)
+            except ParserError:
+                return "The provided start date is not a valid date. Please check the format and try again."
+
+            if parsed_start_date > pendulum.now("UTC"):  # Handle future start date.
+                return "The start date cannot be greater than the current date."
+
+        return None
+
     def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, any]:
+        # First confirm that if start_date is set by user, it is valid.
+        validation_error = self._validate_start_date(config)
+        if validation_error:
+            return False, validation_error
         try:
             authenticator = self._get_authenticator(config)
             stream = Pages(authenticator=authenticator, config=config)

airbyte-integrations/connectors/source-notion/source_notion/spec.json

@@ -4,6 +4,7 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "Notion Source Spec",
     "type": "object",
+    "required": ["credentials"],
     "properties": {
       "start_date": {
         "title": "Start Date",

airbyte-integrations/connectors/source-notion/unit_tests/test_source.py

@@ -35,7 +35,7 @@ def test_check_connection(mocker, requests_mock):
 )
 def test_check_connection_errors(mocker, requests_mock, status_code, json_response, expected_message):
     source = SourceNotion()
-    logger_mock, config_mock = MagicMock(), MagicMock()
+    logger_mock, config_mock = MagicMock(), {"access_token": "test_token", "start_date": "2021-01-01T00:00:00.000Z"}
     requests_mock.post("https://api.notion.com/v1/search", status_code=status_code, json=json_response)
     result, message = source.check_connection(logger_mock, config_mock)
 
@@ -68,3 +68,19 @@ def test_get_authenticator(config, expected_token):
         assert authenticator.token == expected_token  # Replace with the actual way to access the token from the authenticator
     else:
         assert authenticator is None
+
+
+@pytest.mark.parametrize(
+    "config, expected_return",
+    [
+        ({}, None),
+        ({"start_date": "2021-01-01T00:00:00.000Z"}, None),
+        ({"start_date": "2021-99-99T79:89:99.123Z"}, "The provided start date is not a valid date. Please check the format and try again."),
+        ({"start_date": "2021-01-01T00:00:00.000"}, "Please check the format of the start date against the pattern descriptor."),
+        ({"start_date": "2025-01-25T00:00:00.000Z"}, "The start date cannot be greater than the current date."),
+    ],
+)
+def test_validate_start_date(config, expected_return):
+    source = SourceNotion()
+    result = source._validate_start_date(config)
+    assert result == expected_return

docs/integrations/sources/notion.md

@@ -13,9 +13,11 @@ To authenticate the Notion source connector, you need to use **one** of the foll
 - OAuth2.0 authorization (recommended for Airbyte Cloud)
 - Access Token
 
+<!-- env:cloud -->
 :::note
 **For Airbyte Cloud users:** We highly recommend using OAuth2.0 authorization to connect to Notion, as this method significantly simplifies the setup process. If you use OAuth2.0 authorization in Airbyte Cloud, you do **not** need to create and configure a new integration in Notion. Instead, you can proceed straight to [setting up the connector in Airbyte](#step-3-set-up-the-notion-connector-in-airbyte).
 :::
+<!-- /env:cloud -->
 
 We have provided a quick setup guide for creating an integration in Notion below. If you would like more detailed information and context on Notion integrations, or experience any difficulties with the integration setup process, please refer to the [official Notion documentation](https://developers.notion.com/docs).
 
@@ -58,20 +60,24 @@ If you are authenticating via OAuth2.0 for **Airbyte Open Source**, you will nee
 ### Step 3: Set up the Notion connector in Airbyte
 
 1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account, or navigate to your Airbyte Open Source dashboard.
-2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **New source**.
 3. Find and select **Notion** from the list of available sources.
 4. Enter a **Source name** of your choosing.
 5. Choose the method of authentication from the dropdown menu:
 
+<!-- env:cloud -->
 #### Authentication for Airbyte Cloud
 
 - **OAuth2.0** (Recommended): Click **Authenticate your Notion account**. When the popup appears, click **Select pages**. Check the pages you want to give Airbyte access to, and click **Allow access**.
 - **Access Token**: Copy and paste the Access Token found in the **Secrets** tab of your private integration's page.
+<!-- /env:cloud -->
 
+<!-- env:oss -->
 #### Authentication for Airbyte Open Source
 
 - **Access Token**: Copy and paste the Access Token found in the **Secrets** tab of your private integration's page.
 - **OAuth2.0**: Copy and paste the Client ID, Client Secret and Access Token you acquired after setting up your public integration.
+<!-- /env:oss -->
 
 6. (Optional) You may optionally provide a **Start Date** using the provided datepicker, or by programmatically entering a UTC date and time in the format: `YYYY-MM-DDTHH:mm:ss.SSSZ`. When using incremental syncs, only data generated after this date will be replicated. If left blank, Airbyte will set the start date two years from the current date by default.
 7. Click **Set up source** and wait for the tests to complete.
@@ -106,6 +112,7 @@ The connector is restricted by Notion [request limits](https://developers.notion
 
 | Version | Date       | Pull Request                                             | Subject                                                                      |
 | :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------- |
+| 2.0.1   | 2023-10-17 | [31507](https://github.com/airbytehq/airbyte/pull/31507) | Add start_date validation checks                                             |
 | 2.0.0   | 2023-10-09 | [30587](https://github.com/airbytehq/airbyte/pull/30587) | Source-wide schema update                                                    |
 | 1.3.0   | 2023-10-09 | [30324](https://github.com/airbytehq/airbyte/pull/30324) | Add `Comments` stream                                                        |
 | 1.2.2   | 2023-10-09 | [30780](https://github.com/airbytehq/airbyte/pull/30780) | Update Start Date in config to optional field                                |