From 68edf708c84f6d1012f6ad059102a5ee9210c274 Mon Sep 17 00:00:00 2001 From: the-forest-tree Date: Mon, 4 Dec 2023 14:50:26 +0000 Subject: [PATCH] feat: set different foolproof default for jobs and profiles for reading from salesforce --- manifest.json | 12 +++---- .../salesforce/docs/pull_job_list.md | 4 +-- .../salesforce/docs/pull_profile_list.md | 4 +-- .../connectors/salesforce/warehouse.py | 33 ++++++++++++++----- 4 files changed, 35 insertions(+), 18 deletions(-) diff --git a/manifest.json b/manifest.json index fe01d8ced..7bb5eb904 100644 --- a/manifest.json +++ b/manifest.json @@ -19845,7 +19845,7 @@ "trigger_type": "schedule", "origin": "Salesforce Profiles", "origin_parameters": { - "title": "ReadFromSalesforceParameters", + "title": "ReadProfilesParameters", "type": "object", "properties": { "sf_username": { @@ -19880,8 +19880,8 @@ }, "limit": { "title": "Limit", - "description": "Total number of items to pull from Salesforce.By default limiting to 500", - "default": 500, + "description": "Total number of items to pull from Salesforce.By default limiting to 100", + "default": 100, "field_type": "Query Param", "type": "integer" } @@ -22337,7 +22337,7 @@ "trigger_type": "schedule", "origin": "Salesforce Jobs", "origin_parameters": { - "title": "ReadFromSalesforceParameters", + "title": "ReadJobsParameters", "type": "object", "properties": { "sf_username": { @@ -22372,8 +22372,8 @@ }, "limit": { "title": "Limit", - "description": "Total number of items to pull from Salesforce.By default limiting to 500", - "default": 500, + "description": "Total number of items to pull from Salesforce.By default limiting to 1000", + "default": 1000, "field_type": "Query Param", "type": "integer" } diff --git a/src/hrflow_connectors/connectors/salesforce/docs/pull_job_list.md b/src/hrflow_connectors/connectors/salesforce/docs/pull_job_list.md index 61fcc5fcf..18ef1ca99 100644 --- a/src/hrflow_connectors/connectors/salesforce/docs/pull_job_list.md +++ b/src/hrflow_connectors/connectors/salesforce/docs/pull_job_list.md @@ -22,7 +22,7 @@ Retrieves jobs from Salesforce HrFlow Job Custom Object and writes them to an Hr | `sf_security_token` :red_circle: | `str` | None | Security Token to access Salesforce API.See below for instructions: How Can I Find My Security Token and Use It in Data Loader | Salesforce Platform https://www.youtube.com/watch?v=nYbfxeSGKFM&ab_channel=SalesforceSupport | | `sf_organization_id` :red_circle: | `str` | None | See below for instructions: How to find your organization id https://help.salesforce.com/s/articleView?id=000385215&type=1 | | `last_modified_date` | `str` | None | Last modified date | -| `limit` | `int` | 500 | Total number of items to pull from Salesforce.By default limiting to 500 | +| `limit` | `int` | 1000 | Total number of items to pull from Salesforce.By default limiting to 1000 | ## Destination Parameters @@ -61,7 +61,7 @@ Salesforce.pull_job_list( sf_security_token="your_sf_security_token", sf_organization_id="your_sf_organization_id", last_modified_date="your_last_modified_date", - limit=500, + limit=1000, ), target_parameters=dict( api_secret="your_api_secret", diff --git a/src/hrflow_connectors/connectors/salesforce/docs/pull_profile_list.md b/src/hrflow_connectors/connectors/salesforce/docs/pull_profile_list.md index 012ca158d..72a876686 100644 --- a/src/hrflow_connectors/connectors/salesforce/docs/pull_profile_list.md +++ b/src/hrflow_connectors/connectors/salesforce/docs/pull_profile_list.md @@ -22,7 +22,7 @@ Retrieves profiles from Salesforce HrFlow Profile & Co Custom Objects and writes | `sf_security_token` :red_circle: | `str` | None | Security Token to access Salesforce API.See below for instructions: How Can I Find My Security Token and Use It in Data Loader | Salesforce Platform https://www.youtube.com/watch?v=nYbfxeSGKFM&ab_channel=SalesforceSupport | | `sf_organization_id` :red_circle: | `str` | None | See below for instructions: How to find your organization id https://help.salesforce.com/s/articleView?id=000385215&type=1 | | `last_modified_date` | `str` | None | Last modified date | -| `limit` | `int` | 500 | Total number of items to pull from Salesforce.By default limiting to 500 | +| `limit` | `int` | 100 | Total number of items to pull from Salesforce.By default limiting to 100 | ## Destination Parameters @@ -60,7 +60,7 @@ Salesforce.pull_profile_list( sf_security_token="your_sf_security_token", sf_organization_id="your_sf_organization_id", last_modified_date="your_last_modified_date", - limit=500, + limit=100, ), target_parameters=dict( api_secret="your_api_secret", diff --git a/src/hrflow_connectors/connectors/salesforce/warehouse.py b/src/hrflow_connectors/connectors/salesforce/warehouse.py index 095cbf059..0e3c88052 100644 --- a/src/hrflow_connectors/connectors/salesforce/warehouse.py +++ b/src/hrflow_connectors/connectors/salesforce/warehouse.py @@ -21,7 +21,8 @@ WarehouseWriteAction, ) -DEFAULT_LIMIT = 500 +DEFAULT_LIMIT_PROFILES = 100 +DEFAULT_LIMIT_JOBS = 1000 SOQL_MAX_RETURNED_ROWS = 2000 SELECT_PROFILES_SOQL = """ @@ -166,17 +167,33 @@ class SalesforceBaseParameters(ParametersModel): ) -class ReadFromSalesforceParameters(SalesforceBaseParameters): +class ReadProfilesParameters(SalesforceBaseParameters): last_modified_date: t.Optional[str] = Field( None, description="Last modified date", field_type=FieldType.QueryParam, ) limit: int = Field( - DEFAULT_LIMIT, + DEFAULT_LIMIT_PROFILES, description=( "Total number of items to pull from Salesforce." - "By default limiting to {}".format(DEFAULT_LIMIT) + "By default limiting to {}".format(DEFAULT_LIMIT_PROFILES) + ), + field_type=FieldType.QueryParam, + ) + + +class ReadJobsParameters(SalesforceBaseParameters): + last_modified_date: t.Optional[str] = Field( + None, + description="Last modified date", + field_type=FieldType.QueryParam, + ) + limit: int = Field( + DEFAULT_LIMIT_JOBS, + description=( + "Total number of items to pull from Salesforce." + "By default limiting to {}".format(DEFAULT_LIMIT_JOBS) ), field_type=FieldType.QueryParam, ) @@ -190,7 +207,7 @@ def generic_read_factory( ]: def _read_items( adapter: LoggerAdapter, - parameters: ReadFromSalesforceParameters, + parameters: t.Union[ReadProfilesParameters, ReadJobsParameters], read_mode: t.Optional[ReadMode] = None, read_from: t.Optional[str] = None, ) -> t.Iterable[t.Dict]: @@ -265,7 +282,7 @@ def delete_from_salesforce(data: t.List[t.Tuple[SFBulkType, t.List[str]]]) -> No def write_profiles( adapter: LoggerAdapter, - parameters: ReadFromSalesforceParameters, + parameters: SalesforceBaseParameters, profiles: t.Iterable[t.Dict], ) -> t.List[t.Dict]: failed = [] @@ -351,7 +368,7 @@ def item_to_read_from(item: t.Dict) -> str: data_schema=SalesforceHrFlowProfile, data_type=DataType.profile, read=WarehouseReadAction( - parameters=ReadFromSalesforceParameters, + parameters=ReadProfilesParameters, function=generic_read_factory(soql_query=SELECT_PROFILES_SOQL), supports_incremental=True, item_to_read_from=item_to_read_from, @@ -367,7 +384,7 @@ def item_to_read_from(item: t.Dict) -> str: data_schema=SalesforceHrFlowJob, data_type=DataType.job, read=WarehouseReadAction( - parameters=ReadFromSalesforceParameters, + parameters=ReadJobsParameters, function=generic_read_factory(soql_query=SELECT_JOBS_SOQL), supports_incremental=True, item_to_read_from=item_to_read_from,