Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CRM-16865: Mass Dedupe Workflow Improvements #6432

Merged
merged 14 commits into from
Aug 27, 2015
Merged

CRM-16865: Mass Dedupe Workflow Improvements #6432

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
63ef778
Applying patch from Civi v4.5 repo, dedupe-workflow-45 branch, till 9…
deepak-srivastava May 14, 2015
fd630ef
1. flipping dupe pairs for a row or selections. 2. store meaningful -…
deepak-srivastava Jul 21, 2015
da3afd4
get conflict column disaply results with line breaks, and force enabl…
deepak-srivastava Jul 22, 2015
579ea9b
1. escape grid wise search input, 2. correctify search column numbers
deepak-srivastava Jul 22, 2015
f931b74
Code styling fixes
deepak-srivastava Jul 27, 2015
3bfb9d7
button class icons
deepak-srivastava Aug 6, 2015
22b232f
use JSON::output instead of encodeDataTableSelector
deepak-srivastava Aug 8, 2015
183ec33
comments
deepak-srivastava Aug 8, 2015
f2515ec
towards using crmAjaxTable plugin
deepak-srivastava Aug 10, 2015
f826cbf
rowCallback fix
deepak-srivastava Aug 12, 2015
57db188
get rid of crm-pair-conflict, and use index to catch cells to no-wrap
deepak-srivastava Aug 17, 2015
cc86f7b
default page length and class
deepak-srivastava Aug 17, 2015
1f6dc66
Make flip action also flip contact type icons
deepak-srivastava Aug 25, 2015
355927b
disable unsaved-cahnges warnings
deepak-srivastava Aug 25, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
278 changes: 248 additions & 30 deletions CRM/Contact/Page/AJAX.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -668,55 +668,211 @@ public static function processDupes() {
CRM_Utils_JSON::output(array('status' => ($status) ? $oper : $status));
}

/**
* Retrieve list of duplicate pairs from cache table.
*/
public static function getDedupes() {

$sEcho = CRM_Utils_Type::escape($_REQUEST['sEcho'], 'Integer');
$offset = isset($_REQUEST['iDisplayStart']) ? CRM_Utils_Type::escape($_REQUEST['iDisplayStart'], 'Integer') : 0;
$rowCount = isset($_REQUEST['iDisplayLength']) ? CRM_Utils_Type::escape($_REQUEST['iDisplayLength'], 'Integer') : 25;
$sort = 'sort_name';
$sortOrder = isset($_REQUEST['sSortDir_0']) ? CRM_Utils_Type::escape($_REQUEST['sSortDir_0'], 'String') : 'asc';

$gid = isset($_REQUEST['gid']) ? CRM_Utils_Type::escape($_REQUEST['gid'], 'Integer') : 0;
$rgid = isset($_REQUEST['rgid']) ? CRM_Utils_Type::escape($_REQUEST['rgid'], 'Integer') : 0;
$offset = isset($_REQUEST['start']) ? CRM_Utils_Type::escape($_REQUEST['start'], 'Integer') : 0;
$rowCount = isset($_REQUEST['length']) ? CRM_Utils_Type::escape($_REQUEST['length'], 'Integer') : 25;

$gid = isset($_REQUEST['gid']) ? CRM_Utils_Type::escape($_REQUEST['gid'], 'Integer') : 0;
$rgid = isset($_REQUEST['rgid']) ? CRM_Utils_Type::escape($_REQUEST['rgid'], 'Integer') : 0;
$selected = isset($_REQUEST['selected']) ? CRM_Utils_Type::escape($_REQUEST['selected'], 'Integer') : 0;
if ($rowCount < 0) {
$rowCount = 0;
}
$contactType = '';
if ($rgid) {
$contactType = CRM_Core_DAO::getFieldValue('CRM_Dedupe_DAO_RuleGroup', $rgid, 'contact_type');
}

$cacheKeyString = "merge {$contactType}_{$rgid}_{$gid}";
$searchRows = array();
$selectorElements = array('src', 'dst', 'weight', 'actions');
$cacheKeyString = "merge {$contactType}_{$rgid}_{$gid}";
$searchRows = array();
$selectorElements = array('is_selected', 'is_selected_input', 'src_image', 'src', 'src_email', 'src_street', 'src_postcode', 'dst_image', 'dst', 'dst_email', 'dst_street', 'dst_postcode', 'conflicts', 'weight', 'actions');

$join = "LEFT JOIN civicrm_dedupe_exception de ON ( pn.entity_id1 = de.contact_id1 AND
pn.entity_id2 = de.contact_id2 )";
$where = "de.id IS NULL";
foreach ($_REQUEST['columns'] as $columnInfo) {
if (!empty($columnInfo['search']['value'])) {
${$columnInfo['data']} = CRM_Utils_Type::escape($columnInfo['search']['value'], 'String');
}
}
$join = '';
$where = array();
$searchData = CRM_Utils_Array::value('search', $_REQUEST);
$searchData['value'] = CRM_Utils_Type::escape($searchData['value'], 'String');

if ($src || !empty($searchData['value'])) {
$src = $src ? $src : $searchData['value'];
$where[] = " cc1.display_name LIKE '%{$src}%'";
}
if ($dst || !empty($searchData['value'])) {
$dst = $dst ? $dst : $searchData['value'];
$where[] = " cc2.display_name LIKE '%{$dst}%'";
}
if ($src_email || !empty($searchData['value'])) {
$src_email = $src_email ? $src_email : $searchData['value'];
$where[] = " (ce1.is_primary = 1 AND ce1.email LIKE '%{$src_email}%')";
}
if ($dst_email || !empty($searchData['value'])) {
$dst_email = $dst_email ? $dst_email : $searchData['value'];
$where[] = " (ce2.is_primary = 1 AND ce2.email LIKE '%{$dst_email}%')";
}
if ($src_postcode || !empty($searchData['value'])) {
$src_postcode = $src_postcode ? $src_postcode : $searchData['value'];
$where[] = " (ca1.is_primary = 1 AND ca1.postal_code LIKE '%{$src_postcode}%')";
}
if ($dst_postcode || !empty($searchData['value'])) {
$dst_postcode = $dst_postcode ? $dst_postcode : $searchData['value'];
$where[] = " (ca2.is_primary = 1 AND ca2.postal_code LIKE '%{$dst_postcode}%')";
}
if ($src_street || !empty($searchData['value'])) {
$src_street = $src_street ? $src_street : $searchData['value'];
$where[] = " (ca1.is_primary = 1 AND ca1.street_address LIKE '%{$src_street}%')";
}
if ($dst_street || !empty($searchData['value'])) {
$dst_street = $dst_street ? $dst_street : $searchData['value'];
$where[] = " (ca2.is_primary = 1 AND ca2.street_address LIKE '%{$dst_street}%')";
}
if (!empty($searchData['value'])) {
$whereClause = ' ( ' . implode(' OR ', $where) . ' ) ';
}
else {
if (!empty($where)) {
$whereClause = implode(' AND ', $where);
}
}
$whereClause .= $whereClause ? ' AND de.id IS NULL' : ' de.id IS NULL';

$iFilteredTotal = $iTotal = CRM_Core_BAO_PrevNextCache::getCount($cacheKeyString, $join, $where);
$mainContacts = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, $join, $where, $offset, $rowCount);
if ($selected) {
$whereClause .= ' AND pn.is_selected = 1';
}
$join .= " LEFT JOIN civicrm_dedupe_exception de ON ( pn.entity_id1 = de.contact_id1 AND pn.entity_id2 = de.contact_id2 )";

$select = array(
'cc1.contact_type' => 'src_contact_type',
'cc1.display_name' => 'src_display_name',
'cc1.contact_sub_type' => 'src_contact_sub_type',
'cc2.contact_type' => 'dst_contact_type',
'cc2.display_name' => 'dst_display_name',
'cc2.contact_sub_type' => 'dst_contact_sub_type',
'ce1.email' => 'src_email',
'ce2.email' => 'dst_email',
'ca1.postal_code' => 'src_postcode',
'ca2.postal_code' => 'dst_postcode',
'ca1.street_address' => 'src_street',
'ca2.street_address' => 'dst_street',
);

if ($select) {
$join .= " INNER JOIN civicrm_contact cc1 ON cc1.id = pn.entity_id1";
$join .= " INNER JOIN civicrm_contact cc2 ON cc2.id = pn.entity_id2";
$join .= " LEFT JOIN civicrm_email ce1 ON (ce1.contact_id = pn.entity_id1 AND ce1.is_primary = 1 )";
$join .= " LEFT JOIN civicrm_email ce2 ON (ce2.contact_id = pn.entity_id2 AND ce2.is_primary = 1 )";
$join .= " LEFT JOIN civicrm_address ca1 ON (ca1.contact_id = pn.entity_id1 AND ca1.is_primary = 1 )";
$join .= " LEFT JOIN civicrm_address ca2 ON (ca2.contact_id = pn.entity_id2 AND ca2.is_primary = 1 )";
}
$iTotal = CRM_Core_BAO_PrevNextCache::getCount($cacheKeyString, $join, $whereClause);
foreach ($_REQUEST['order'] as $orderInfo) {
if (!empty($orderInfo['column'])) {
$orderColumnNumber = $orderInfo['column'];
$dir = $orderInfo['dir'];
}
}
$columnDetails = CRM_Utils_Array::value($orderColumnNumber, $_REQUEST['columns']);
if (!empty($columnDetails)) {
switch ($columnDetails['data']) {
case 'src':
$whereClause .= " ORDER BY cc1.display_name {$dir}";
break;

case 'src_email':
$whereClause .= " ORDER BY ce1.email {$dir}";
break;

case 'src_street':
$whereClause .= " ORDER BY ca1.street_address {$dir}";
break;

case 'src_postcode':
$whereClause .= " ORDER BY ca1.postal_code {$dir}";
break;

case 'dst':
$whereClause .= " ORDER BY cc2.display_name {$dir}";
break;

case 'dst_email':
$whereClause .= " ORDER BY ce2.email {$dir}";
break;

case 'dst_street':
$whereClause .= " ORDER BY ca2.street_address {$dir}";
break;

case 'dst_postcode':
$whereClause .= " ORDER BY ca2.postal_code {$dir}";
break;

default:
break;
}
}

foreach ($mainContacts as $mainId => $main) {
$searchRows[$mainId]['src'] = CRM_Utils_System::href($main['srcName'], 'civicrm/contact/view', "reset=1&cid={$main['srcID']}");
$searchRows[$mainId]['dst'] = CRM_Utils_System::href($main['dstName'], 'civicrm/contact/view', "reset=1&cid={$main['dstID']}");
$searchRows[$mainId]['weight'] = CRM_Utils_Array::value('weight', $main);
$dupePairs = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, $join, $whereClause, $offset, $rowCount, $select);
$iFilteredTotal = CRM_Core_DAO::singleValueQuery("SELECT FOUND_ROWS()");

$count = 0;
foreach ($dupePairs as $key => $pairInfo) {
$pair =& $pairInfo['data'];
$srcContactSubType = CRM_Utils_Array::value('src_contact_sub_type', $pairInfo);
$dstContactSubType = CRM_Utils_Array::value('dst_contact_sub_type', $pairInfo);
$srcTypeImage = CRM_Contact_BAO_Contact_Utils::getImage($srcContactSubType ?
$srcContactSubType : $pairInfo['src_contact_type'],
FALSE,
$pairInfo['entity_id1']
);
$dstTypeImage = CRM_Contact_BAO_Contact_Utils::getImage($dstContactSubType ?
$dstContactSubType : $pairInfo['dst_contact_type'],
FALSE,
$pairInfo['entity_id2']
);

if (!empty($main['canMerge'])) {
$mergeParams = "reset=1&cid={$main['srcID']}&oid={$main['dstID']}&action=update&rgid={$rgid}";
$searchRows[$count]['is_selected'] = $pairInfo['is_selected'];
$searchRows[$count]['is_selected_input'] = "<input type='checkbox' class='crm-dedupe-select' name='pnid_{$pairInfo['prevnext_id']}' value='{$pairInfo['is_selected']}' onclick='toggleDedupeSelect(this)'>";
$searchRows[$count]['src_image'] = $srcTypeImage;
$searchRows[$count]['src'] = CRM_Utils_System::href($pair['srcName'], 'civicrm/contact/view', "reset=1&cid={$pairInfo['entity_id1']}");
$searchRows[$count]['src_email'] = CRM_Utils_Array::value('src_email', $pairInfo);
$searchRows[$count]['src_street'] = CRM_Utils_Array::value('src_street', $pairInfo);
$searchRows[$count]['src_postcode'] = CRM_Utils_Array::value('src_postcode', $pairInfo);
$searchRows[$count]['dst_image'] = $dstTypeImage;
$searchRows[$count]['dst'] = CRM_Utils_System::href($pair['dstName'], 'civicrm/contact/view', "reset=1&cid={$pairInfo['entity_id2']}");
$searchRows[$count]['dst_email'] = CRM_Utils_Array::value('dst_email', $pairInfo);
$searchRows[$count]['dst_street'] = CRM_Utils_Array::value('dst_street', $pairInfo);
$searchRows[$count]['dst_postcode'] = CRM_Utils_Array::value('dst_postcode', $pairInfo);
$searchRows[$count]['conflicts'] = str_replace("',", "',<br/>", CRM_Utils_Array::value('conflicts', $pair));
$searchRows[$count]['weight'] = CRM_Utils_Array::value('weight', $pair);

if (!empty($pairInfo['data']['canMerge'])) {
$mergeParams = "reset=1&cid={$pairInfo['entity_id1']}&oid={$pairInfo['entity_id2']}&action=update&rgid={$rgid}";
if ($gid) {
$mergeParams .= "&gid={$gid}";
}

$searchRows[$mainId]['actions'] = '<a class="action-item crm-hover-button" href="' . CRM_Utils_System::url('civicrm/contact/merge', $mergeParams) . '">' . ts('merge') . '</a>';
$searchRows[$mainId]['actions'] .= "<a class='action-item crm-hover-button crm-notDuplicate' href='#' onClick=\"processDupes( {$main['srcID']}, {$main['dstID']}, 'dupe-nondupe', 'dupe-listing'); return false;\">" . ts('not a duplicate') . "</a>";
$searchRows[$count]['actions'] = "<a class='crm-dedupe-flip' href='#' data-pnid={$pairInfo['prevnext_id']}>" . ts('flip') . "</a>&nbsp;|&nbsp;";
$searchRows[$count]['actions'] .= CRM_Utils_System::href(ts('merge'), 'civicrm/contact/merge', $mergeParams);
$searchRows[$count]['actions'] .= "&nbsp;|&nbsp;<a id='notDuplicate' href='#' onClick=\"processDupes( {$pairInfo['entity_id1']}, {$pairInfo['entity_id2']}, 'dupe-nondupe', 'dupe-listing'); return false;\">" . ts('not a duplicate') . "</a>";
}
else {
$searchRows[$mainId]['actions'] = '<em>' . ts('Insufficient access rights - cannot merge') . '</em>';
$searchRows[$count]['actions'] = '<em>' . ts('Insufficient access rights - cannot merge') . '</em>';
}
$count++;
}

CRM_Utils_System::setHttpHeader('Content-Type', 'application/json');
echo CRM_Utils_JSON::encodeDataTableSelector($searchRows, $sEcho, $iTotal, $iFilteredTotal, $selectorElements);

CRM_Utils_System::civiExit();
$dupePairs = array(
'data' => $searchRows,
'recordsTotal' => $iTotal,
'recordsFiltered' => $iFilteredTotal,
);
CRM_Utils_JSON::output($dupePairs);
}

/**
Expand All @@ -741,6 +897,30 @@ public static function paperSize() {
CRM_Utils_JSON::output($paperSize);
}

/**
* Swap contacts in a dupe pair i.e main with duplicate contact.
*/
public static function flipDupePairs($prevNextId = NULL) {
if (!$prevNextId) {
$prevNextId = $_REQUEST['pnid'];
}
$query = "
UPDATE civicrm_prevnext_cache cpc
INNER JOIN civicrm_prevnext_cache old on cpc.id = old.id
SET cpc.entity_id1 = cpc.entity_id2, cpc.entity_id2 = old.entity_id1 ";
if (is_array($prevNextId) && !CRM_Utils_Array::crmIsEmptyArray($prevNextId)) {
$prevNextId = implode(', ', $prevNextId);
$prevNextId = CRM_Utils_Type::escape($prevNextId, 'String');
$query .= "WHERE cpc.id IN ({$prevNextId}) AND cpc.is_selected = 1";
}
else {
$prevNextId = CRM_Utils_Type::escape($prevNextId, 'Positive');
$query .= "WHERE cpc.id = $prevNextId";
}
CRM_Core_DAO::executeQuery($query);
CRM_Utils_JSON::output();
}

/**
* Used to store selected contacts across multiple pages in advanced search.
*/
Expand Down Expand Up @@ -806,6 +986,44 @@ public static function getAddressDisplay() {
CRM_Utils_JSON::output($addressVal);
}

/**
* Mark dupe pairs as selected from un-selected state or vice-versa, in dupe cache table.
*/
public static function toggleDedupeSelect() {
$rgid = CRM_Utils_Type::escape($_REQUEST['rgid'], 'Integer');
$gid = CRM_Utils_Type::escape($_REQUEST['gid'], 'Integer');
$pnid = $_REQUEST['pnid'];
$isSelected = CRM_Utils_Type::escape($_REQUEST['is_selected'], 'Boolean');

$contactType = CRM_Core_DAO::getFieldValue('CRM_Dedupe_DAO_RuleGroup', $rgid, 'contact_type');
$cacheKeyString = "merge $contactType";
$cacheKeyString .= $rgid ? "_{$rgid}" : '_0';
$cacheKeyString .= $gid ? "_{$gid}" : '_0';

$params = array(
1 => array($isSelected, 'Boolean'),
3 => array("$cacheKeyString%", 'String'), // using % to address rows with conflicts as well
);

//check pnid is_array or integer
$whereClause = NULL;
if (is_array($pnid) && !CRM_Utils_Array::crmIsEmptyArray($pnid)) {
$pnid = implode(', ', $pnid);
$pnid = CRM_Utils_Type::escape($pnid, 'String');
$whereClause = " id IN ( {$pnid} ) ";
}
else {
$pnid = CRM_Utils_Type::escape($pnid, 'Integer');
$whereClause = " id = %2";
$params[2] = array($pnid, 'Integer');
}

$sql = "UPDATE civicrm_prevnext_cache SET is_selected = %1 WHERE {$whereClause} AND cacheKey LIKE %3";
CRM_Core_DAO::executeQuery($sql, $params);

CRM_Utils_System::civiExit();
}

/**
* Retrieve contact relationships.
*/
Expand Down
25 changes: 24 additions & 1 deletion CRM/Contact/Page/DedupeFind.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ public function run() {
elseif ($action & CRM_Core_Action::MAP) {
// do a batch merge if requested
$rgid = CRM_Utils_Request::retrieve('rgid', 'Positive', $this, FALSE, 0);
$result = CRM_Dedupe_Merger::batchMerge($rgid, $gid, 'safe', TRUE, TRUE);
$result = CRM_Dedupe_Merger::batchMerge($rgid, $gid, 'safe', TRUE, 75);

$skippedCount = CRM_Utils_Request::retrieve('skipped', 'Positive', $this, FALSE, 0);
$skippedCount = $skippedCount + count($result['skipped']);
Expand Down Expand Up @@ -157,6 +157,9 @@ public function run() {
if ($rgid) {
$sourceParams .= "&rgid={$rgid}";
}
if ($context == 'conflicts') {
$sourceParams .= "&selected=1";
}

$this->assign('sourceUrl', CRM_Utils_System::url('civicrm/ajax/dedupefind', $sourceParams, FALSE, NULL, FALSE));

Expand All @@ -165,11 +168,31 @@ public function run() {
$cacheKeyString .= $rgid ? "_{$rgid}" : '_0';
$cacheKeyString .= $gid ? "_{$gid}" : '_0';

$stats = CRM_Dedupe_Merger::getMergeStatsMsg($cacheKeyString);
if ($stats) {
CRM_Core_Session::setStatus($stats);
// reset so we not displaying same message again
CRM_Dedupe_Merger::resetMergeStats($cacheKeyString);
}
$join = "LEFT JOIN civicrm_dedupe_exception de ON ( pn.entity_id1 = de.contact_id1 AND
pn.entity_id2 = de.contact_id2 )";
$where = "de.id IS NULL";
if ($context == 'conflicts') {
$where .= " AND pn.is_selected = 1";
}
$this->_mainContacts = CRM_Core_BAO_PrevNextCache::retrieve($cacheKeyString, $join, $where);
if (empty($this->_mainContacts)) {
if ($context == 'conflicts') {
// if the current screen was intended to list only selected contacts, move back to full dupe list
$sourceParams = 'reset=1&action=update';
if ($gid) {
$sourceParams .= "&gid={$gid}";
}
if ($rgid) {
$sourceParams .= "&rgid={$rgid}";
}
CRM_Utils_System::redirect(CRM_Utils_System::url(CRM_Utils_System::currentPath(), $sourceParams));
}
if ($gid) {
$foundDupes = $this->get("dedupe_dupes_$gid");
if (!$foundDupes) {
Expand Down
Loading