feat: set different foolproof default for jobs and profiles for reading

from salesforce
Riminder · Dec 4, 2023 · 68edf70 · 68edf70
1 parent b5bdb1a
commit 68edf70
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 18 deletions.
diff --git a/manifest.json b/manifest.json
@@ -19845,7 +19845,7 @@
           "trigger_type": "schedule",
           "origin": "Salesforce Profiles",
           "origin_parameters": {
-            "title": "ReadFromSalesforceParameters",
+            "title": "ReadProfilesParameters",
             "type": "object",
             "properties": {
               "sf_username": {
@@ -19880,8 +19880,8 @@
               },
               "limit": {
                 "title": "Limit",
-                "description": "Total number of items to pull from Salesforce.By default limiting to 500",
-                "default": 500,
+                "description": "Total number of items to pull from Salesforce.By default limiting to 100",
+                "default": 100,
                 "field_type": "Query Param",
                 "type": "integer"
               }
@@ -22337,7 +22337,7 @@
           "trigger_type": "schedule",
           "origin": "Salesforce Jobs",
           "origin_parameters": {
-            "title": "ReadFromSalesforceParameters",
+            "title": "ReadJobsParameters",
             "type": "object",
             "properties": {
               "sf_username": {
@@ -22372,8 +22372,8 @@
               },
               "limit": {
                 "title": "Limit",
-                "description": "Total number of items to pull from Salesforce.By default limiting to 500",
-                "default": 500,
+                "description": "Total number of items to pull from Salesforce.By default limiting to 1000",
+                "default": 1000,
                 "field_type": "Query Param",
                 "type": "integer"
               }

diff --git a/src/hrflow_connectors/connectors/salesforce/docs/pull_job_list.md b/src/hrflow_connectors/connectors/salesforce/docs/pull_job_list.md
@@ -22,7 +22,7 @@ Retrieves jobs from Salesforce HrFlow Job Custom Object and writes them to an Hr
 | `sf_security_token` :red_circle: | `str` | None | Security Token to access Salesforce API.See below for instructions: How Can I Find My Security Token and Use It in Data Loader | Salesforce Platform  https://www.youtube.com/watch?v=nYbfxeSGKFM&ab_channel=SalesforceSupport |
 | `sf_organization_id` :red_circle: | `str` | None | See below for instructions: How to find your organization id  https://help.salesforce.com/s/articleView?id=000385215&type=1 |
 | `last_modified_date`  | `str` | None | Last modified date |
-| `limit`  | `int` | 500 | Total number of items to pull from Salesforce.By default limiting to 500 |
+| `limit`  | `int` | 1000 | Total number of items to pull from Salesforce.By default limiting to 1000 |
 
 ## Destination Parameters
 
@@ -61,7 +61,7 @@ Salesforce.pull_job_list(
         sf_security_token="your_sf_security_token",
         sf_organization_id="your_sf_organization_id",
         last_modified_date="your_last_modified_date",
-        limit=500,
+        limit=1000,
     ),
     target_parameters=dict(
         api_secret="your_api_secret",

diff --git a/src/hrflow_connectors/connectors/salesforce/docs/pull_profile_list.md b/src/hrflow_connectors/connectors/salesforce/docs/pull_profile_list.md
@@ -22,7 +22,7 @@ Retrieves profiles from Salesforce HrFlow Profile & Co Custom Objects and writes
 | `sf_security_token` :red_circle: | `str` | None | Security Token to access Salesforce API.See below for instructions: How Can I Find My Security Token and Use It in Data Loader | Salesforce Platform  https://www.youtube.com/watch?v=nYbfxeSGKFM&ab_channel=SalesforceSupport |
 | `sf_organization_id` :red_circle: | `str` | None | See below for instructions: How to find your organization id  https://help.salesforce.com/s/articleView?id=000385215&type=1 |
 | `last_modified_date`  | `str` | None | Last modified date |
-| `limit`  | `int` | 500 | Total number of items to pull from Salesforce.By default limiting to 500 |
+| `limit`  | `int` | 100 | Total number of items to pull from Salesforce.By default limiting to 100 |
 
 ## Destination Parameters
 
@@ -60,7 +60,7 @@ Salesforce.pull_profile_list(
         sf_security_token="your_sf_security_token",
         sf_organization_id="your_sf_organization_id",
         last_modified_date="your_last_modified_date",
-        limit=500,
+        limit=100,
     ),
     target_parameters=dict(
         api_secret="your_api_secret",

diff --git a/src/hrflow_connectors/connectors/salesforce/warehouse.py b/src/hrflow_connectors/connectors/salesforce/warehouse.py
@@ -21,7 +21,8 @@
     WarehouseWriteAction,
 )
 
-DEFAULT_LIMIT = 500
+DEFAULT_LIMIT_PROFILES = 100
+DEFAULT_LIMIT_JOBS = 1000
 SOQL_MAX_RETURNED_ROWS = 2000
 
 SELECT_PROFILES_SOQL = """
@@ -166,17 +167,33 @@ class SalesforceBaseParameters(ParametersModel):
     )
 
 
-class ReadFromSalesforceParameters(SalesforceBaseParameters):
+class ReadProfilesParameters(SalesforceBaseParameters):
     last_modified_date: t.Optional[str] = Field(
         None,
         description="Last modified date",
         field_type=FieldType.QueryParam,
     )
     limit: int = Field(
-        DEFAULT_LIMIT,
+        DEFAULT_LIMIT_PROFILES,
         description=(
             "Total number of items to pull from Salesforce."
-            "By default limiting to {}".format(DEFAULT_LIMIT)
+            "By default limiting to {}".format(DEFAULT_LIMIT_PROFILES)
+        ),
+        field_type=FieldType.QueryParam,
+    )
+
+
+class ReadJobsParameters(SalesforceBaseParameters):
+    last_modified_date: t.Optional[str] = Field(
+        None,
+        description="Last modified date",
+        field_type=FieldType.QueryParam,
+    )
+    limit: int = Field(
+        DEFAULT_LIMIT_JOBS,
+        description=(
+            "Total number of items to pull from Salesforce."
+            "By default limiting to {}".format(DEFAULT_LIMIT_JOBS)
         ),
         field_type=FieldType.QueryParam,
     )
@@ -190,7 +207,7 @@ def generic_read_factory(
 ]:
     def _read_items(
         adapter: LoggerAdapter,
-        parameters: ReadFromSalesforceParameters,
+        parameters: t.Union[ReadProfilesParameters, ReadJobsParameters],
         read_mode: t.Optional[ReadMode] = None,
         read_from: t.Optional[str] = None,
     ) -> t.Iterable[t.Dict]:
@@ -265,7 +282,7 @@ def delete_from_salesforce(data: t.List[t.Tuple[SFBulkType, t.List[str]]]) -> No
 
 def write_profiles(
     adapter: LoggerAdapter,
-    parameters: ReadFromSalesforceParameters,
+    parameters: SalesforceBaseParameters,
     profiles: t.Iterable[t.Dict],
 ) -> t.List[t.Dict]:
     failed = []
@@ -351,7 +368,7 @@ def item_to_read_from(item: t.Dict) -> str:
     data_schema=SalesforceHrFlowProfile,
     data_type=DataType.profile,
     read=WarehouseReadAction(
-        parameters=ReadFromSalesforceParameters,
+        parameters=ReadProfilesParameters,
         function=generic_read_factory(soql_query=SELECT_PROFILES_SOQL),
         supports_incremental=True,
         item_to_read_from=item_to_read_from,
@@ -367,7 +384,7 @@ def item_to_read_from(item: t.Dict) -> str:
     data_schema=SalesforceHrFlowJob,
     data_type=DataType.job,
     read=WarehouseReadAction(
-        parameters=ReadFromSalesforceParameters,
+        parameters=ReadJobsParameters,
         function=generic_read_factory(soql_query=SELECT_JOBS_SOQL),
         supports_incremental=True,
         item_to_read_from=item_to_read_from,