diff --git a/CRM/Contact/Form/Merge.php b/CRM/Contact/Form/Merge.php index 91f282f6bb4e..da804021ea2c 100644 --- a/CRM/Contact/Form/Merge.php +++ b/CRM/Contact/Form/Merge.php @@ -114,7 +114,7 @@ public function preProcess() { CRM_Core_Session::singleton()->pushUserContext($browseUrl); } - $cacheKey = CRM_Dedupe_Merger::getMergeCacheKeyString($this->_rgid, $gid, json_decode($this->criteria, TRUE)); + $cacheKey = CRM_Dedupe_Merger::getMergeCacheKeyString($this->_rgid, $gid, json_decode($this->criteria, TRUE), TRUE, $this->limit); $join = CRM_Dedupe_Merger::getJoinOnDedupeTable(); $where = "de.id IS NULL"; @@ -331,7 +331,7 @@ public function postProcess() { } if ($this->next && $this->_mergeId) { - $cacheKey = CRM_Dedupe_Merger::getMergeCacheKeyString($this->_rgid, $this->_gid, json_decode($this->criteria, TRUE)); + $cacheKey = CRM_Dedupe_Merger::getMergeCacheKeyString($this->_rgid, $this->_gid, json_decode($this->criteria, TRUE), TRUE, $this->limit); $join = CRM_Dedupe_Merger::getJoinOnDedupeTable(); $where = "de.id IS NULL"; diff --git a/CRM/Contact/Page/AJAX.php b/CRM/Contact/Page/AJAX.php index 7dcedcc8cd5f..3ac0326cf743 100644 --- a/CRM/Contact/Page/AJAX.php +++ b/CRM/Contact/Page/AJAX.php @@ -638,6 +638,7 @@ public static function getDedupes() { $gid = CRM_Utils_Request::retrieve('gid', 'Positive'); $rgid = CRM_Utils_Request::retrieve('rgid', 'Positive'); + $limit = CRM_Utils_Request::retrieveValue('limit', 'Positive', 0); $null = NULL; $criteria = CRM_Utils_Request::retrieve('criteria', 'Json', $null, FALSE, '{}'); $selected = CRM_Utils_Request::retrieveValue('selected', 'Boolean'); @@ -646,7 +647,7 @@ public static function getDedupes() { } $whereClause = $orderByClause = ''; - $cacheKeyString = CRM_Dedupe_Merger::getMergeCacheKeyString($rgid, $gid, json_decode($criteria, TRUE)); + $cacheKeyString = CRM_Dedupe_Merger::getMergeCacheKeyString($rgid, $gid, json_decode($criteria, TRUE), TRUE, $limit); $searchRows = []; diff --git a/CRM/Contact/Page/DedupeFind.php b/CRM/Contact/Page/DedupeFind.php index 8b22a7140d92..b7a22ea66611 100644 --- a/CRM/Contact/Page/DedupeFind.php +++ b/CRM/Contact/Page/DedupeFind.php @@ -105,13 +105,13 @@ public function run() { 'reset' => 1, 'rgid' => $rgid, 'gid' => $gid, - 'limit' => $limit, + 'limit' => (int) $limit, 'criteria' => $criteria, ]; $this->assign('urlQuery', CRM_Utils_System::makeQueryString($urlQry)); $this->assign('isSelected', $this->isSelected()); $criteria = json_decode($criteria, TRUE); - $cacheKeyString = CRM_Dedupe_Merger::getMergeCacheKeyString($rgid, $gid, $criteria); + $cacheKeyString = CRM_Dedupe_Merger::getMergeCacheKeyString($rgid, $gid, $criteria, TRUE, $limit); $this->assign('cacheKey', $cacheKeyString); if ($context == 'search') { @@ -129,7 +129,7 @@ public function run() { } elseif ($action & CRM_Core_Action::MAP) { // do a batch merge if requested - $result = CRM_Dedupe_Merger::batchMerge($rgid, $gid, 'safe', 75, 2, $criteria); + $result = CRM_Dedupe_Merger::batchMerge($rgid, $gid, 'safe', 75, 2, $criteria, TRUE, NULL, $limit); $skippedCount = CRM_Utils_Request::retrieve('skipped', 'Positive', $this, FALSE, 0); $skippedCount = $skippedCount + count($result['skipped']); diff --git a/CRM/Contact/Page/DedupeMerge.php b/CRM/Contact/Page/DedupeMerge.php index 1dddf5619470..348f65b1d8f8 100644 --- a/CRM/Contact/Page/DedupeMerge.php +++ b/CRM/Contact/Page/DedupeMerge.php @@ -70,7 +70,7 @@ public static function getRunner() { ]; $criteria = json_decode($criteria, TRUE); - $cacheKeyString = CRM_Dedupe_Merger::getMergeCacheKeyString($rgid, $gid, $criteria); + $cacheKeyString = CRM_Dedupe_Merger::getMergeCacheKeyString($rgid, $gid, $criteria, TRUE, $limit); if ($mode == 'aggressive' && !CRM_Core_Permission::check('force merge duplicate contacts')) { CRM_Core_Session::setStatus(ts('You do not have permission to force merge duplicate contact records'), ts('Permission Denied'), 'error'); @@ -98,7 +98,7 @@ public static function getRunner() { for ($i = 1; $i <= ceil($total / self::BATCHLIMIT); $i++) { $task = new CRM_Queue_Task( ['CRM_Contact_Page_DedupeMerge', 'callBatchMerge'], - [$rgid, $gid, $mode, self::BATCHLIMIT, $onlyProcessSelected, $criteria], + [$rgid, $gid, $mode, self::BATCHLIMIT, $onlyProcessSelected, $criteria, $limit], "Processed " . $i * self::BATCHLIMIT . " pair of duplicates out of " . $total ); @@ -132,11 +132,15 @@ public static function getRunner() { * @param int $batchLimit * @param int $isSelected * @param array $criteria + * @param int $searchLimit * * @return int + * + * @throws \CRM_Core_Exception + * @throws \CiviCRM_API3_Exception */ - public static function callBatchMerge(CRM_Queue_TaskContext $ctx, $rgid, $gid, $mode = 'safe', $batchLimit, $isSelected, $criteria) { - CRM_Dedupe_Merger::batchMerge($rgid, $gid, $mode, $batchLimit, $isSelected, $criteria, TRUE, FALSE); + public static function callBatchMerge(CRM_Queue_TaskContext $ctx, $rgid, $gid, $mode = 'safe', $batchLimit, $isSelected, $criteria, $searchLimit) { + CRM_Dedupe_Merger::batchMerge($rgid, $gid, $mode, $batchLimit, $isSelected, $criteria, TRUE, FALSE, $searchLimit); return CRM_Queue_Task::TASK_SUCCESS; } diff --git a/CRM/Core/BAO/PrevNextCache.php b/CRM/Core/BAO/PrevNextCache.php index bcc8e650f8f0..20b9fcfe486b 100644 --- a/CRM/Core/BAO/PrevNextCache.php +++ b/CRM/Core/BAO/PrevNextCache.php @@ -429,7 +429,7 @@ public static function getCount($cacheKey, $join = NULL, $where = NULL, $op = "= * @throws \CiviCRM_API3_Exception */ public static function refillCache($rgid, $gid, $criteria, $checkPermissions, $searchLimit = 0) { - $cacheKeyString = CRM_Dedupe_Merger::getMergeCacheKeyString($rgid, $gid, $criteria, $checkPermissions); + $cacheKeyString = CRM_Dedupe_Merger::getMergeCacheKeyString($rgid, $gid, $criteria, $checkPermissions, $searchLimit); // 1. Clear cache if any $sql = "DELETE FROM civicrm_prevnext_cache WHERE cachekey LIKE %1"; diff --git a/CRM/Dedupe/Merger.php b/CRM/Dedupe/Merger.php index 9e501f35f60c..a35ae67e792d 100644 --- a/CRM/Dedupe/Merger.php +++ b/CRM/Dedupe/Merger.php @@ -691,9 +691,17 @@ public static function retrieveFields($main, $other) { * If not set explicitly this is calculated but it is preferred that it be set * per comments on isSelected above. * + * @param int $searchLimit + * Limit on number of contacts to search for duplicates for. + * This means that if the limit is 1000 then only duplicates for the first 1000 contacts + * matching criteria will be found and batchMerged (the number of merges could be less than or greater than 100) + * * @return array|bool + * + * @throws \CRM_Core_Exception + * @throws \CiviCRM_API3_Exception */ - public static function batchMerge($rgid, $gid = NULL, $mode = 'safe', $batchLimit = 1, $isSelected = 2, $criteria = [], $checkPermissions = TRUE, $reloadCacheIfEmpty = NULL) { + public static function batchMerge($rgid, $gid = NULL, $mode = 'safe', $batchLimit = 1, $isSelected = 2, $criteria = [], $checkPermissions = TRUE, $reloadCacheIfEmpty = NULL, $searchLimit = 0) { $redirectForPerformance = ($batchLimit > 1) ? TRUE : FALSE; if (!isset($reloadCacheIfEmpty)) { @@ -706,7 +714,7 @@ public static function batchMerge($rgid, $gid = NULL, $mode = 'safe', $batchLimi $dupePairs = self::getDuplicatePairs($rgid, $gid, $reloadCacheIfEmpty, $batchLimit, $isSelected, ($mode == 'aggressive'), $criteria, $checkPermissions); $cacheParams = [ - 'cache_key_string' => self::getMergeCacheKeyString($rgid, $gid, $criteria, $checkPermissions), + 'cache_key_string' => self::getMergeCacheKeyString($rgid, $gid, $criteria, $checkPermissions, $searchLimit), // @todo stop passing these parameters in & instead calculate them in the merge function based // on the 'real' params like $isRespectExclusions $batchLimit and $isSelected. 'join' => self::getJoinOnDedupeTable(), @@ -1840,13 +1848,13 @@ public static function createMergeActivities($mainId, $otherId) { * @throws \CiviCRM_API3_Exception */ public static function getDuplicatePairs($rule_group_id, $group_id, $reloadCacheIfEmpty, $batchLimit, $isSelected, $includeConflicts = TRUE, $criteria = [], $checkPermissions = TRUE, $searchLimit = 0) { - $dupePairs = self::getCachedDuplicateMatches($rule_group_id, $group_id, $batchLimit, $isSelected, $includeConflicts, $criteria, $checkPermissions); + $dupePairs = self::getCachedDuplicateMatches($rule_group_id, $group_id, $batchLimit, $isSelected, $includeConflicts, $criteria, $checkPermissions, $searchLimit); if (empty($dupePairs) && $reloadCacheIfEmpty) { // If we haven't found any dupes, probably cache is empty. // Try filling cache and give another try. We don't need to specify include conflicts here are there will not be any // until we have done some processing. CRM_Core_BAO_PrevNextCache::refillCache($rule_group_id, $group_id, $criteria, $checkPermissions, $searchLimit); - return self::getCachedDuplicateMatches($rule_group_id, $group_id, $batchLimit, $isSelected, FALSE, $criteria, $checkPermissions); + return self::getCachedDuplicateMatches($rule_group_id, $group_id, $batchLimit, $isSelected, FALSE, $criteria, $checkPermissions, $searchLimit); } return $dupePairs; } @@ -1859,17 +1867,21 @@ public static function getDuplicatePairs($rule_group_id, $group_id, $reloadCache * @param array $criteria * Additional criteria to narrow down the merge group. * Currently we are only supporting the key 'contact' within it. - * * @param bool $checkPermissions * Respect the users permissions. + * @param int $searchLimit + * Number of contacts to seek dupes for (we need this because if + * we change it the results won't be refreshed otherwise. Changing the limit + * from 100 to 1000 SHOULD result in a new dedupe search). * * @return string */ - public static function getMergeCacheKeyString($rule_group_id, $group_id, $criteria = [], $checkPermissions = TRUE) { + public static function getMergeCacheKeyString($rule_group_id, $group_id, $criteria, $checkPermissions, $searchLimit) { $contactType = CRM_Dedupe_BAO_RuleGroup::getContactTypeForRuleGroup($rule_group_id); $cacheKeyString = "merge_{$contactType}"; $cacheKeyString .= $rule_group_id ? "_{$rule_group_id}" : '_0'; $cacheKeyString .= $group_id ? "_{$group_id}" : '_0'; + $cacheKeyString .= '_' . (int) $searchLimit; $cacheKeyString .= !empty($criteria) ? md5(serialize($criteria)) : '_0'; if ($checkPermissions) { $contactID = CRM_Core_Session::getLoggedInContactID(); @@ -2487,12 +2499,13 @@ protected static function formatConflictArray($conflicts, $migrationInfo, $toKee * @param bool $includeConflicts * @param array $criteria * @param int $checkPermissions + * @param int $searchLimit * * @return array */ - protected static function getCachedDuplicateMatches($rule_group_id, $group_id, $batchLimit, $isSelected, $includeConflicts, $criteria, $checkPermissions) { + protected static function getCachedDuplicateMatches($rule_group_id, $group_id, $batchLimit, $isSelected, $includeConflicts, $criteria, $checkPermissions, $searchLimit = 0) { return CRM_Core_BAO_PrevNextCache::retrieve( - self::getMergeCacheKeyString($rule_group_id, $group_id, $criteria, $checkPermissions), + self::getMergeCacheKeyString($rule_group_id, $group_id, $criteria, $checkPermissions, $searchLimit), self::getJoinOnDedupeTable(), self::getWhereString($isSelected), 0, $batchLimit, diff --git a/api/v3/Dedupe.php b/api/v3/Dedupe.php index 009920691840..58041ea3f948 100644 --- a/api/v3/Dedupe.php +++ b/api/v3/Dedupe.php @@ -107,7 +107,8 @@ function civicrm_api3_dedupe_getstatistics($params) { $params['rule_group_id'], CRM_Utils_Array::value('group_id', $params), CRM_Utils_Array::value('criteria', $params, []), - CRM_Utils_Array::value('check_permissions', $params, []) + !empty($params['check_permissions']), + CRM_Utils_Array::value('search_limit', $params, 0) )); return civicrm_api3_create_success($stats); } @@ -135,6 +136,11 @@ function _civicrm_api3_dedupe_getstatistics_spec(&$params) { 'title' => ts('Criteria'), 'description' => ts('Dedupe search criteria, as parsable by v3 Contact.get api'), ]; + $spec['search_limit'] = [ + 'title' => ts('Number of contacts to look for matches for.'), + 'type' => CRM_Utils_Type::T_INT, + 'api.default' => (int) Civi::settings()->get('dedupe_default_limit'), + ]; } diff --git a/api/v3/Job.php b/api/v3/Job.php index 86cfa253d27f..4a4b0eca75f0 100644 --- a/api/v3/Job.php +++ b/api/v3/Job.php @@ -541,7 +541,7 @@ function civicrm_api3_job_process_batch_merge($params) { $gid = CRM_Utils_Array::value('gid', $params); $mode = CRM_Utils_Array::value('mode', $params, 'safe'); - $result = CRM_Dedupe_Merger::batchMerge($rule_group_id, $gid, $mode, 1, 2, CRM_Utils_Array::value('criteria', $params, []), CRM_Utils_Array::value('check_permissions', $params)); + $result = CRM_Dedupe_Merger::batchMerge($rule_group_id, $gid, $mode, 1, 2, CRM_Utils_Array::value('criteria', $params, []), CRM_Utils_Array::value('check_permissions', $params), NULL, $params['search_limit']); return civicrm_api3_create_success($result, $params); } @@ -571,6 +571,12 @@ function _civicrm_api3_job_process_batch_merge_spec(&$params) { 'description' => 'let the api decide which contact to retain and which to delete?', 'type' => CRM_Utils_Type::T_BOOLEAN, ]; + $params['search_limit'] = [ + 'title' => ts('Number of contacts to look for matches for.'), + 'type' => CRM_Utils_Type::T_INT, + 'api.default' => (int) Civi::settings()->get('dedupe_default_limit'), + ]; + } /** diff --git a/tests/phpunit/CRM/Dedupe/MergerTest.php b/tests/phpunit/CRM/Dedupe/MergerTest.php index a5e3c8c67a24..0798109eae5b 100644 --- a/tests/phpunit/CRM/Dedupe/MergerTest.php +++ b/tests/phpunit/CRM/Dedupe/MergerTest.php @@ -175,7 +175,7 @@ public function testBatchMergeSelectedDuplicates() { // Retrieve pairs from prev next cache table $select = ['pn.is_selected' => 'is_selected']; - $cacheKeyString = CRM_Dedupe_Merger::getMergeCacheKeyString($dao->id, $this->_groupId); + $cacheKeyString = CRM_Dedupe_Merger::getMergeCacheKeyString($dao->id, $this->_groupId, [], TRUE, 0); $pnDupePairs = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, NULL, NULL, 0, 0, $select); $this->assertEquals(count($foundDupes), count($pnDupePairs), 'Check number of dupe pairs in prev next cache.'); @@ -245,7 +245,7 @@ public function testBatchMergeAllDuplicates() { // Retrieve pairs from prev next cache table $select = ['pn.is_selected' => 'is_selected']; - $cacheKeyString = CRM_Dedupe_Merger::getMergeCacheKeyString($dao->id, $this->_groupId); + $cacheKeyString = CRM_Dedupe_Merger::getMergeCacheKeyString($dao->id, $this->_groupId, [], TRUE, 0); $pnDupePairs = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, NULL, NULL, 0, 0, $select); $this->assertEquals(count($foundDupes), count($pnDupePairs), 'Check number of dupe pairs in prev next cache.');