Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Contact import - extract common code, make tags & groups queue-friendly #23680

Merged
merged 6 commits into from
Jun 3, 2022
Merged

Contact import - extract common code, make tags & groups queue-friendly #23680

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d6bc16d
Extract common part of MapTable.tpl
eileenmcnaughton Jun 3, 2022
4d9f4d6
Contact import - extract assignments to be used by other forms
eileenmcnaughton Jun 3, 2022
578e4db
Fix flow for groups & tags
eileenmcnaughton Jun 2, 2022
4a07230
Add generic label get function - not used yet
eileenmcnaughton Jun 3, 2022
8307831
Move setImportStatus to parent
eileenmcnaughton Jun 3, 2022
8d0967f
Add generic PreProcess
eileenmcnaughton Jun 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 2 additions & 11 deletions CRM/Contact/Import/Form/MapField.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,17 +101,8 @@ public function preProcess() {
// retrieve and highlight required custom fields
$formattedFieldNames = $this->formatCustomFieldName($this->_mapperFields);

$this->assign('highlightedFields', $this->getHighlightedFields());
$this->_formattedFieldNames[$contactType] = $this->_mapperFields = array_merge($this->_mapperFields, $formattedFieldNames);

$columnNames = $this->getColumnHeaders();

$this->_columnCount = $this->getNumberOfColumns();
$this->_columnNames = $columnNames;
$this->assign('columnNames', $this->getColumnHeaders());
$this->assign('columnCount', $this->_columnCount);
$this->_dataValues = array_values($this->getDataRows([], 2));
$this->assign('dataValues', $this->_dataValues);
$this->assignMapFieldVariables();
}

/**
Expand Down Expand Up @@ -512,7 +503,7 @@ private function isSkipDuplicates(): bool {
*
* @throws \CRM_Core_Exception
*/
private function getHighlightedFields(): array {
protected function getHighlightedFields(): array {
$entityFields = [
'Individual' => ['first_name', 'last_name'],
'Organization' => ['organization_name'],
Expand Down
158 changes: 83 additions & 75 deletions CRM/Contact/Import/Form/Preview.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@
* @copyright CiviCRM LLC https://civicrm.org/licensing
*/

use Civi\Api4\Group;
use Civi\Api4\Tag;

/**
* This class previews the uploaded file and returns summary statistics.
*/
Expand All @@ -34,38 +37,8 @@ class CRM_Contact_Import_Form_Preview extends CRM_Import_Form_Preview {
* @throws \CRM_Core_Exception
*/
public function preProcess() {
$columnNames = $this->getColumnHeaders();
parent::preProcess();
$this->_disableUSPS = $this->getSubmittedValue('disableUSPS');

//assign column names
$this->assign('columnNames', $columnNames);

//get the mapping name displayed if the mappingId is set
$mappingId = $this->get('loadMappingId');
if ($mappingId) {
$mapDAO = new CRM_Core_DAO_Mapping();
$mapDAO->id = $mappingId;
$mapDAO->find(TRUE);
}
$this->assign('savedMappingName', $mappingId ? $mapDAO->name : NULL);

$this->assign('rowDisplayCount', 2);

$groups = CRM_Core_PseudoConstant::nestedGroup();
$this->set('groups', $groups);

$tag = CRM_Core_PseudoConstant::get('CRM_Core_DAO_EntityTag', 'tag_id', array('onlyActive' => FALSE));
if ($tag) {
$this->set('tag', $tag);
}

$this->assign('downloadErrorRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::ERROR));
$this->assign('invalidRowCount', $this->getRowCount(CRM_Import_Parser::ERROR));
$this->assign('validRowCount', $this->getRowCount(CRM_Import_Parser::VALID));
$this->assign('totalRowCount', $this->getRowCount([]));
$this->assign('mapper', $this->getMappedFieldLabels());
$this->assign('dataValues', $this->getDataRows([], 2));

$this->setStatusUrl();
}

Expand All @@ -84,24 +57,25 @@ public function buildQuickForm() {
);
}

$groups = $this->get('groups');
$groups = CRM_Core_PseudoConstant::nestedGroup();;

if (!empty($groups)) {
$this->addElement('select', 'groups', ts('Add imported records to existing group(s)'), $groups, array(
$this->addElement('select', 'groups', ts('Add imported records to existing group(s)'), $groups, [
'multiple' => "multiple",
'class' => 'crm-select2',
));
]);
}

//display new tag
$this->addElement('text', 'newTagName', ts('Tag'), CRM_Core_DAO::getAttribute('CRM_Core_DAO_Tag', 'name'));
$this->addElement('text', 'newTagDesc', ts('Description'), CRM_Core_DAO::getAttribute('CRM_Core_DAO_Tag', 'description'));

$tag = $this->get('tag');
$tag = CRM_Core_PseudoConstant::get('CRM_Core_DAO_EntityTag', 'tag_id', ['onlyActive' => FALSE]);
if (!empty($tag)) {
foreach ($tag as $tagID => $tagName) {
$this->addElement('checkbox', "tag[$tagID]", NULL, $tagName);
}
$this->addElement('select', 'tag', ts(' Tag imported records'), $tag, [
'multiple' => 'multiple',
'class' => 'crm-select2',
]);
}

$this->addFormRule(array('CRM_Contact_Import_Form_Preview', 'formRule'), $this);
Expand Down Expand Up @@ -163,37 +137,71 @@ public static function formRule($fields, $files, $self) {
/**
* Process the mapped fields and map it into the uploaded file.
*
* @throws \API_Exception
* @throws \API_Exception|\CRM_Core_Exception
*/
public function postProcess() {

$importJobParams = array(
'doGeocodeAddress' => $this->getSubmittedValue('doGeocodeAddress'),
'invalidRowCount' => $this->getRowCount(CRM_Import_Parser::ERROR),
'onDuplicate' => $this->getSubmittedValue('onDuplicate'),
'dedupe' => $this->getSubmittedValue('dedupe_rule_id'),
'newGroupName' => $this->controller->exportValue($this->_name, 'newGroupName'),
'newGroupDesc' => $this->controller->exportValue($this->_name, 'newGroupDesc'),
'newGroupType' => $this->controller->exportValue($this->_name, 'newGroupType'),
'groups' => $this->controller->exportValue($this->_name, 'groups'),
'allGroups' => $this->get('groups'),
'newTagName' => $this->controller->exportValue($this->_name, 'newTagName'),
'newTagDesc' => $this->controller->exportValue($this->_name, 'newTagDesc'),
'tag' => $this->controller->exportValue($this->_name, 'tag'),
'allTags' => $this->get('tag'),
'mapper' => $this->controller->exportValue('MapField', 'mapper'),
'mapFields' => $this->getAvailableFields(),
'contactType' => $this->getContactType(),
'contactSubType' => $this->getSubmittedValue('contactSubType'),
'primaryKeyName' => '_id',
'statusFieldName' => '_status',
'statusID' => $this->get('statusID'),
'totalRowCount' => $this->getRowCount([]),
'userJobID' => $this->getUserJobID(),
);
public function postProcess(): void {
$groupsToAddTo = (array) $this->getSubmittedValue('groups');
$summaryInfo = ['groups' => [], 'tags' => []];
foreach ($groupsToAddTo as $groupID) {
// This is a convenience for now - really url & name should be determined at
// presentation stage - ie the summary screen. The only info we are really
// preserving is which groups were created vs already existed.
$summaryInfo['groups'][$groupID] = [
'url' => CRM_Utils_System::url('civicrm/group/search', 'reset=1&force=1&context=smog&gid=' . $groupID),
'name' => Group::get(FALSE)->addWhere('id', '=', $groupID)->addSelect('name')->execute()->first()['name'],
'new' => FALSE,
'added' => 0,
'notAdded' => 0,
];
}

$importJob = new CRM_Contact_Import_ImportJob();
$importJob->setJobParams($importJobParams);
if ($this->getSubmittedValue('newGroupName')) {
/* Create a new group */
$groupsToAddTo[] = $groupID = Group::create(FALSE)->setValues([
'title' => $this->getSubmittedValue('newGroupName'),
'description' => $this->getSubmittedValue('newGroupDesc'),
'group_type' => $this->getSubmittedValue('newGroupType') ?? [],
'is_active' => TRUE,
])->execute()->first()['id'];
$summaryInfo['groups'][$groupID] = [
'url' => CRM_Utils_System::url('civicrm/group/search', 'reset=1&force=1&context=smog&gid=' . $groupID),
'name' => $this->getSubmittedValue('newGroupName'),
'new' => TRUE,
'added' => 0,
'notAdded' => 0,
];
}
$tagsToAdd = (array) $this->getSubmittedValue('tag');
foreach ($tagsToAdd as $tagID) {
// This is a convenience for now - really url & name should be determined at
// presentation stage - ie the summary screen. The only info we are really
// preserving is which tags were created vs already existed.
$summaryInfo['tags'][$tagID] = [
'url' => CRM_Utils_System::url('civicrm/contact/search', 'reset=1&force=1&context=smog&id=' . $tagID),
'name' => Tag::get(FALSE)->addWhere('id', '=', $tagID)->addSelect('name')->execute()->first()['name'],
'new' => TRUE,
'added' => 0,
'notAdded' => 0,
];
}
if ($this->getSubmittedValue('newTagName')) {
$tagsToAdd[] = $tagID = Tag::create(FALSE)->setValues([
'name' => $this->getSubmittedValue('newTagName'),
'description' => $this->getSubmittedValue('newTagDesc'),
'is_selectable' => TRUE,
'used_for' => 'civicrm_contact',
])->execute()->first()['id'];
$summaryInfo['tags'][$tagID] = [
'url' => CRM_Utils_System::url('civicrm/contact/search', 'reset=1&force=1&context=smog&id=' . $tagID),
'name' => $this->getSubmittedValue('newTagName'),
'new' => FALSE,
'added' => 0,
'notAdded' => 0,
];
}
// Store the actions to take on each row & the data to present at the end to the userJob.
$this->updateUserJobMetadata('post_actions', ['group' => $groupsToAddTo, 'tag' => $tagsToAdd]);
$this->updateUserJobMetadata('summary_info', $summaryInfo);

// If ACL applies to the current user, update cache before running the import.
if (!CRM_Core_Permission::check('view all contacts')) {
Expand All @@ -205,16 +213,20 @@ public function postProcess() {
CRM_Utils_Address_USPS::disable($this->_disableUSPS);

// run the import
$importJob->runImport($this);

$this->_parser = $this->getParser();
$this->_parser->run(
[],
CRM_Import_Parser::MODE_IMPORT,
$this->get('statusID')
);

// Clear all caches, forcing any searches to recheck the ACLs or group membership as the import
// may have changed it.
CRM_Contact_BAO_Contact_Utils::clearContactCaches(TRUE);

// add all the necessary variables to the form
$importJob->setFormVariables($this);

// check if there is any error occurred
// @todo - it's really unclear that this error code should still exist...
$errorStack = CRM_Core_Error::singleton();
$errors = $errorStack->getErrors();
$errorMessage = [];
Expand All @@ -235,10 +247,6 @@ public function postProcess() {

$this->set('errorFile', $errorFile);
}

//hack to clean db
//if job complete drop table.
$importJob->isComplete();
}

/**
Expand Down
7 changes: 4 additions & 3 deletions CRM/Contact/Import/Form/Summary.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ class CRM_Contact_Import_Form_Summary extends CRM_Import_Form_Summary {
* @throws \CRM_Core_Exception
*/
public function preProcess() {
// set the error message path to display
// @todo - totally unclear that this errorFile could ever be set / render.
// Probably it can go.
$this->assign('errorFile', $this->get('errorFile'));
$onDuplicate = $this->getSubmittedValue('onDuplicate');
$this->assign('dupeError', FALSE);
Expand All @@ -44,8 +45,8 @@ public function preProcess() {
$this->assign('dupeError', TRUE);
}

$this->assign('groupAdditions', $this->get('groupAdditions'));
$this->assign('tagAdditions', $this->get('tagAdditions'));
$this->assign('groupAdditions', $this->getUserJob()['metadata']['summary_info']['groups']);
$this->assign('tagAdditions', $this->getUserJob()['metadata']['summary_info']['tags']);
$this->assign('totalRowCount', $this->getRowCount());
$this->assign('validRowCount', $this->getRowCount(CRM_Import_Parser::VALID) + $this->getRowCount(CRM_Import_Parser::UNPARSED_ADDRESS_WARNING));
$this->assign('invalidRowCount', $this->getRowCount(CRM_Import_Parser::ERROR));
Expand Down
Loading