feat: foolproof limit by setting a default to 500

Riminder · Dec 4, 2023 · b5bdb1a · b5bdb1a
1 parent 773c8f2
commit b5bdb1a
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 13 deletions.
diff --git a/manifest.json b/manifest.json
@@ -19868,7 +19868,7 @@
               },
               "sf_organization_id": {
                 "title": "Sf Organization Id",
-                "description": "Security Token to access Salesforce API.See below for instructions: How to find your organization id  https://help.salesforce.com/s/articleView?id=000385215&type=1",
+                "description": "See below for instructions: How to find your organization id  https://help.salesforce.com/s/articleView?id=000385215&type=1",
                 "field_type": "Auth",
                 "type": "string"
               },
@@ -19880,7 +19880,7 @@
               },
               "limit": {
                 "title": "Limit",
-                "description": "Number of items to pull from Salesforce. Maximum value is 500",
+                "description": "Total number of items to pull from Salesforce.By default limiting to 500",
                 "default": 500,
                 "field_type": "Query Param",
                 "type": "integer"
@@ -21628,7 +21628,7 @@
               },
               "sf_organization_id": {
                 "title": "Sf Organization Id",
-                "description": "Security Token to access Salesforce API.See below for instructions: How to find your organization id  https://help.salesforce.com/s/articleView?id=000385215&type=1",
+                "description": "See below for instructions: How to find your organization id  https://help.salesforce.com/s/articleView?id=000385215&type=1",
                 "field_type": "Auth",
                 "type": "string"
               }
@@ -22360,7 +22360,7 @@
               },
               "sf_organization_id": {
                 "title": "Sf Organization Id",
-                "description": "Security Token to access Salesforce API.See below for instructions: How to find your organization id  https://help.salesforce.com/s/articleView?id=000385215&type=1",
+                "description": "See below for instructions: How to find your organization id  https://help.salesforce.com/s/articleView?id=000385215&type=1",
                 "field_type": "Auth",
                 "type": "string"
               },
@@ -22372,7 +22372,7 @@
               },
               "limit": {
                 "title": "Limit",
-                "description": "Number of items to pull from Salesforce. Maximum value is 500",
+                "description": "Total number of items to pull from Salesforce.By default limiting to 500",
                 "default": 500,
                 "field_type": "Query Param",
                 "type": "integer"

diff --git a/src/hrflow_connectors/connectors/salesforce/docs/pull_job_list.md b/src/hrflow_connectors/connectors/salesforce/docs/pull_job_list.md
@@ -22,7 +22,7 @@ Retrieves jobs from Salesforce HrFlow Job Custom Object and writes them to an Hr
 | `sf_security_token` :red_circle: | `str` | None | Security Token to access Salesforce API.See below for instructions: How Can I Find My Security Token and Use It in Data Loader | Salesforce Platform  https://www.youtube.com/watch?v=nYbfxeSGKFM&ab_channel=SalesforceSupport |
 | `sf_organization_id` :red_circle: | `str` | None | See below for instructions: How to find your organization id  https://help.salesforce.com/s/articleView?id=000385215&type=1 |
 | `last_modified_date`  | `str` | None | Last modified date |
-| `limit`  | `int` | None | Total number of items to pull from Salesforce |
+| `limit`  | `int` | 500 | Total number of items to pull from Salesforce.By default limiting to 500 |
 
 ## Destination Parameters
 
@@ -61,7 +61,7 @@ Salesforce.pull_job_list(
         sf_security_token="your_sf_security_token",
         sf_organization_id="your_sf_organization_id",
         last_modified_date="your_last_modified_date",
-        limit=0,
+        limit=500,
     ),
     target_parameters=dict(
         api_secret="your_api_secret",

diff --git a/src/hrflow_connectors/connectors/salesforce/docs/pull_profile_list.md b/src/hrflow_connectors/connectors/salesforce/docs/pull_profile_list.md
@@ -22,7 +22,7 @@ Retrieves profiles from Salesforce HrFlow Profile & Co Custom Objects and writes
 | `sf_security_token` :red_circle: | `str` | None | Security Token to access Salesforce API.See below for instructions: How Can I Find My Security Token and Use It in Data Loader | Salesforce Platform  https://www.youtube.com/watch?v=nYbfxeSGKFM&ab_channel=SalesforceSupport |
 | `sf_organization_id` :red_circle: | `str` | None | See below for instructions: How to find your organization id  https://help.salesforce.com/s/articleView?id=000385215&type=1 |
 | `last_modified_date`  | `str` | None | Last modified date |
-| `limit`  | `int` | None | Total number of items to pull from Salesforce |
+| `limit`  | `int` | 500 | Total number of items to pull from Salesforce.By default limiting to 500 |
 
 ## Destination Parameters
 
@@ -60,7 +60,7 @@ Salesforce.pull_profile_list(
         sf_security_token="your_sf_security_token",
         sf_organization_id="your_sf_organization_id",
         last_modified_date="your_last_modified_date",
-        limit=0,
+        limit=500,
     ),
     target_parameters=dict(
         api_secret="your_api_secret",

diff --git a/src/hrflow_connectors/connectors/salesforce/warehouse.py b/src/hrflow_connectors/connectors/salesforce/warehouse.py
@@ -21,6 +21,7 @@
     WarehouseWriteAction,
 )
 
+DEFAULT_LIMIT = 500
 SOQL_MAX_RETURNED_ROWS = 2000
 
 SELECT_PROFILES_SOQL = """
@@ -171,9 +172,12 @@ class ReadFromSalesforceParameters(SalesforceBaseParameters):
         description="Last modified date",
         field_type=FieldType.QueryParam,
     )
-    limit: t.Optional[int] = Field(
-        None,
-        description="Total number of items to pull from Salesforce",
+    limit: int = Field(
+        DEFAULT_LIMIT,
+        description=(
+            "Total number of items to pull from Salesforce."
+            "By default limiting to {}".format(DEFAULT_LIMIT)
+        ),
         field_type=FieldType.QueryParam,
     )
 
@@ -228,7 +232,7 @@ def _read_items(
                 )
                 last_id = 0
 
-        remaining = parameters.limit or float("inf")
+        remaining = parameters.limit
         query = soql_query.format(
             last_modified_date=last_modified_date,
             limit_placeholder=(