diff --git a/CRM/Contact/Import/Form/Summary.php b/CRM/Contact/Import/Form/Summary.php index c8adb53bfe50..84b7c3b922f0 100644 --- a/CRM/Contact/Import/Form/Summary.php +++ b/CRM/Contact/Import/Form/Summary.php @@ -48,16 +48,7 @@ public function preProcess() { $this->assign('groupAdditions', $this->getUserJob()['metadata']['summary_info']['groups']); $this->assign('tagAdditions', $this->getUserJob()['metadata']['summary_info']['tags']); - $this->assign('totalRowCount', $this->getRowCount()); - $this->assign('validRowCount', $this->getRowCount(CRM_Import_Parser::VALID) + $this->getRowCount(CRM_Import_Parser::UNPARSED_ADDRESS_WARNING)); - $this->assign('invalidRowCount', $this->getRowCount(CRM_Import_Parser::ERROR)); - $this->assign('duplicateRowCount', $this->getRowCount(CRM_Import_Parser::DUPLICATE)); - $this->assign('unMatchCount', $this->getRowCount(CRM_Import_Parser::NO_MATCH)); - $this->assign('unparsedAddressCount', $this->getRowCount(CRM_Import_Parser::UNPARSED_ADDRESS_WARNING)); - $this->assign('downloadDuplicateRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::DUPLICATE)); - $this->assign('downloadErrorRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::ERROR)); - $this->assign('downloadMismatchRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::NO_MATCH)); - $this->assign('downloadAddressRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::UNPARSED_ADDRESS_WARNING)); + $this->assignOutputURLs(); $session = CRM_Core_Session::singleton(); $session->pushUserContext(CRM_Utils_System::url('civicrm/import/contact', 'reset=1')); } diff --git a/CRM/Contact/Import/Parser/Contact.php b/CRM/Contact/Import/Parser/Contact.php index a1b98e4b71cc..83777cc25794 100644 --- a/CRM/Contact/Import/Parser/Contact.php +++ b/CRM/Contact/Import/Parser/Contact.php @@ -77,6 +77,8 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser { * The initializer code, called before processing. */ public function init() { + // Force re-load of user job. + unset($this->userJob); $this->setFieldMetadata(); foreach ($this->getImportableFieldsMetadata() as $name => $field) { $this->addField($name, $field['title'], CRM_Utils_Array::value('type', $field), CRM_Utils_Array::value('headerPattern', $field), CRM_Utils_Array::value('dataPattern', $field), CRM_Utils_Array::value('hasLocationType', $field)); diff --git a/CRM/Import/Form/DataSource.php b/CRM/Import/Form/DataSource.php index a5e53ad43fd8..306fe2d75523 100644 --- a/CRM/Import/Form/DataSource.php +++ b/CRM/Import/Form/DataSource.php @@ -158,7 +158,6 @@ protected function storeFormValues($names) { * Entity to set for paraser currently only for custom import */ protected function submitFileForMapping($parserClassName, $entity = NULL) { - $this->controller->resetPage('MapField'); CRM_Core_Session::singleton()->set('dateTypes', $this->getSubmittedValue('dateFormats')); $this->processDatasource(); @@ -181,6 +180,7 @@ protected function submitFileForMapping($parserClassName, $entity = NULL) { // add all the necessary variables to the form $parser->set($this); + $this->controller->resetPage('MapField'); } /** diff --git a/CRM/Import/Form/MapField.php b/CRM/Import/Form/MapField.php index 11581f2a4699..26f90307782b 100644 --- a/CRM/Import/Form/MapField.php +++ b/CRM/Import/Form/MapField.php @@ -76,6 +76,22 @@ public function preProcess() { parent::preProcess(); } + /** + * Process the mapped fields and map it into the uploaded file + * preview the file and extract some summary statistics + * + * @return void + * @noinspection PhpDocSignatureInspection + * @noinspection PhpUnhandledExceptionInspection + */ + public function postProcess() { + $this->updateUserJobMetadata('submitted_values', $this->getSubmittedValues()); + $this->saveMapping($this->getMappingTypeName()); + $parser = $this->getParser(); + $parser->init(); + $parser->validate(); + } + /** * Attempt to match header labels with our mapper fields. * @@ -230,6 +246,9 @@ protected function getMappedField(array $fieldMapping, int $mappingID, int $colu protected function saveMappingField(int $mappingID, int $columnNumber, bool $isUpdate = FALSE): void { $fieldMapping = (array) $this->getSubmittedValue('mapper')[$columnNumber]; $mappedField = $this->getMappedField($fieldMapping, $mappingID, $columnNumber); + if (empty($mappedField['name'])) { + $mappedField['name'] = 'do_not_import'; + } if ($isUpdate) { Civi\Api4\MappingField::update(FALSE) ->setValues($mappedField) diff --git a/CRM/Import/Form/Preview.php b/CRM/Import/Form/Preview.php index cce75402e928..69e302d8c99a 100644 --- a/CRM/Import/Form/Preview.php +++ b/CRM/Import/Form/Preview.php @@ -117,4 +117,14 @@ protected function assignPreviewVariables(): void { $this->assign('rowDisplayCount', $this->getSubmittedValue('skipColumnHeader') ? 3 : 2); } + /** + * Process the mapped fields and map it into the uploaded file + * preview the file and extract some summary statistics + * + * @return void + */ + public function postProcess() { + CRM_Import_Parser::runImport(NULL, $this->getUserJobID(), 0); + } + } diff --git a/CRM/Import/Form/Summary.php b/CRM/Import/Form/Summary.php index b27af617eea1..72c8be2f2744 100644 --- a/CRM/Import/Form/Summary.php +++ b/CRM/Import/Form/Summary.php @@ -23,6 +23,15 @@ */ abstract class CRM_Import_Form_Summary extends CRM_Import_Forms { + /** + * Set variables up before form is built. + * + * @return void + */ + public function preProcess() { + $this->assignOutputURLs(); + } + /** * Build the form object. */ @@ -45,4 +54,17 @@ public function getTitle() { return ts('Summary'); } + protected function assignOutputURLs(): void { + $this->assign('totalRowCount', $this->getRowCount()); + $this->assign('validRowCount', $this->getRowCount(CRM_Import_Parser::VALID) + $this->getRowCount(CRM_Import_Parser::UNPARSED_ADDRESS_WARNING)); + $this->assign('invalidRowCount', $this->getRowCount(CRM_Import_Parser::ERROR)); + $this->assign('duplicateRowCount', $this->getRowCount(CRM_Import_Parser::DUPLICATE)); + $this->assign('unMatchCount', $this->getRowCount(CRM_Import_Parser::NO_MATCH)); + $this->assign('unparsedAddressCount', $this->getRowCount(CRM_Import_Parser::UNPARSED_ADDRESS_WARNING)); + $this->assign('downloadDuplicateRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::DUPLICATE)); + $this->assign('downloadErrorRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::ERROR)); + $this->assign('downloadMismatchRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::NO_MATCH)); + $this->assign('downloadAddressRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::UNPARSED_ADDRESS_WARNING)); + } + } diff --git a/CRM/Import/Parser.php b/CRM/Import/Parser.php index 30b8c8ae5ff4..d02d9c1f47f0 100644 --- a/CRM/Import/Parser.php +++ b/CRM/Import/Parser.php @@ -635,8 +635,8 @@ protected function validateRequiredContactFields(string $contactType, array $par protected function doPostImportActions() { $userJob = $this->getUserJob(); - $summaryInfo = $userJob['metadata']['summary_info']; - $actions = $userJob['metadata']['post_actions']; + $summaryInfo = $userJob['metadata']['summary_info'] ?? []; + $actions = $userJob['metadata']['post_actions'] ?? []; if (!empty($actions['group'])) { $groupAdditions = $this->addImportedContactsToNewGroup($this->createdContacts, $actions['group']); foreach ($actions['group'] as $groupID) { @@ -1575,7 +1575,8 @@ public function validate(): void { protected function getInvalidValues($value, string $key = '', string $prefixString = ''): array { $errors = []; if ($value === 'invalid_import_value') { - $errors[] = $prefixString . $this->getFieldMetadata($key)['title']; + $metadata = $this->getFieldMetadata($key); + $errors[] = $prefixString . ($metadata['html']['label'] ?? $metadata['title']); } elseif (is_array($value)) { foreach ($value as $innerKey => $innerValue) { @@ -1666,6 +1667,40 @@ public function getMappingFieldFromMapperInput(array $fieldMapping, int $mapping ]; } + /** + * @param array $mappedField + * Field detail as would be saved in field_mapping table + * or as returned from getMappingFieldFromMapperInput + * + * @return string + * @throws \API_Exception + */ + public function getMappedFieldLabel(array $mappedField): string { + $this->setFieldMetadata(); + return $this->getFieldMetadata($mappedField['name'])['title']; + } + + /** + * Get the row from the csv mapped to our parameters. + * + * @param array $values + * + * @return array + * @throws \API_Exception + */ + public function getMappedRow(array $values): array { + $params = []; + foreach ($this->getFieldMappings() as $i => $mappedField) { + if ($mappedField['name'] === 'do_not_import') { + continue; + } + if ($mappedField['name']) { + $params[$this->getFieldMetadata($mappedField['name'])['name']] = $this->getTransformedFieldValue($mappedField['name'], $values[$i]); + } + } + return $params; + } + /** * Get the field mappings for the import. * @@ -1678,7 +1713,8 @@ public function getMappingFieldFromMapperInput(array $fieldMapping, int $mapping */ protected function getFieldMappings(): array { $mappedFields = []; - foreach ($this->getSubmittedValue('mapper') as $i => $mapperRow) { + $mapper = $this->getSubmittedValue('mapper'); + foreach ($mapper as $i => $mapperRow) { $mappedField = $this->getMappingFieldFromMapperInput($mapperRow, 0, $i); // Just for clarity since 0 is a pseudo-value unset($mappedField['mapping_id']); @@ -1807,8 +1843,8 @@ protected function getSubtypes($contactType) { * @param int|null $entityID * Optional created entity ID * - * @throws \API_Exception - * @throws \CRM_Core_Exception + * @noinspection PhpDocMissingThrowsInspection + * @noinspection PhpUnhandledExceptionInspection */ protected function setImportStatus(int $id, string $status, string $message, ?int $entityID = NULL): void { $this->getDataSourceObject()->updateStatus($id, $status, $message, $entityID); diff --git a/CRM/Member/Import/Form/MapField.php b/CRM/Member/Import/Form/MapField.php index b5cccd9dfc67..ef105204e9ec 100644 --- a/CRM/Member/Import/Form/MapField.php +++ b/CRM/Member/Import/Form/MapField.php @@ -20,139 +20,28 @@ */ class CRM_Member_Import_Form_MapField extends CRM_Import_Form_MapField { - /** - * store contactType. - * - * @var int - */ - public static $_contactType = NULL; - - /** - * Set variables up before form is built. - * - * @return void - */ - public function preProcess() { - $this->_mapperFields = $this->get('fields'); - asort($this->_mapperFields); - - $this->_columnCount = $this->get('columnCount'); - $this->assign('columnCount', $this->_columnCount); - $this->_dataValues = $this->get('dataValues'); - $this->assign('dataValues', $this->_dataValues); - - $skipColumnHeader = $this->controller->exportValue('DataSource', 'skipColumnHeader'); - $this->_onDuplicate = $this->get('onDuplicate', $onDuplicate ?? ""); - - $highlightedFields = []; - if ($skipColumnHeader) { - $this->assign('skipColumnHeader', $skipColumnHeader); - $this->assign('rowDisplayCount', 3); - /* if we had a column header to skip, stash it for later */ - - $this->_columnHeaders = $this->_dataValues[0]; - } - else { - $this->assign('rowDisplayCount', 2); - } - - //CRM-2219 removing other required fields since for updation only - //membership id is required. - if ($this->_onDuplicate == CRM_Import_Parser::DUPLICATE_UPDATE) { - $remove = array('membership_contact_id', 'email', 'first_name', 'last_name', 'external_identifier'); - foreach ($remove as $value) { - unset($this->_mapperFields[$value]); - } - $highlightedFieldsArray = array('membership_id', 'membership_start_date', 'membership_type_id'); - foreach ($highlightedFieldsArray as $name) { - $highlightedFields[] = $name; - } - } - elseif ($this->_onDuplicate == CRM_Import_Parser::DUPLICATE_SKIP) { - unset($this->_mapperFields['membership_id']); - $highlightedFieldsArray = array( - 'membership_contact_id', - 'email', - 'external_identifier', - 'membership_start_date', - 'membership_type_id', - ); - foreach ($highlightedFieldsArray as $name) { - $highlightedFields[] = $name; - } - } - - // modify field title - $this->_mapperFields['status_id'] = ts('Membership Status'); - $this->_mapperFields['membership_type_id'] = ts('Membership Type'); - - self::$_contactType = $this->get('contactType'); - $this->assign('highlightedFields', $highlightedFields); - } - /** * Build the form object. * * @return void */ public function buildQuickForm() { - //to save the current mappings - if (!$this->get('savedMapping')) { - $saveDetailsName = ts('Save this field mapping'); - $this->applyFilter('saveMappingName', 'trim'); - $this->add('text', 'saveMappingName', ts('Name')); - $this->add('text', 'saveMappingDesc', ts('Description')); - } - else { - $savedMapping = $this->get('savedMapping'); - - list($mappingName) = CRM_Core_BAO_Mapping::getMappingFields($savedMapping); - - $mappingName = $mappingName[1]; - - //mapping is to be loaded from database - - $this->set('loadedMapping', $savedMapping); - - $getMappingName = new CRM_Core_DAO_Mapping(); - $getMappingName->id = $savedMapping; - $getMappingName->mapping_type = 'Import Memberships'; - $getMappingName->find(); - while ($getMappingName->fetch()) { - $mapperName = $getMappingName->name; - } - - $this->assign('savedMappingName', $mapperName); - - $this->add('hidden', 'mappingId', $savedMapping); - - $this->addElement('checkbox', 'updateMapping', ts('Update this field mapping'), NULL); - $saveDetailsName = ts('Save as a new field mapping'); - $this->add('text', 'saveMappingName', ts('Name')); - $this->add('text', 'saveMappingDesc', ts('Description')); - } - - $this->addElement('checkbox', 'saveMapping', $saveDetailsName, NULL, array('onclick' => "showSaveDetails(this)")); - + $this->buildSavedMappingFields($this->getSubmittedValue('savedMapping')); $this->addFormRule(array('CRM_Member_Import_Form_MapField', 'formRule'), $this); //-------- end of saved mapping stuff --------- $defaults = []; - $mapperKeys = array_keys($this->_mapperFields); - $hasHeaders = !empty($this->_columnHeaders); - $headerPatterns = $this->get('headerPatterns'); - $dataPatterns = $this->get('dataPatterns'); - - /* Initialize all field usages to false */ - - foreach ($mapperKeys as $key) { - $this->_fieldUsed[$key] = FALSE; - } - $this->_location_types = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Address', 'location_type_id'); + $columnHeaders = $this->getColumnHeaders(); + $hasHeaders = $this->getSubmittedValue('skipColumnHeader'); + $headerPatterns = $this->getHeaderPatterns(); + $dataPatterns = $this->getDataPatterns(); + // For most fields using the html label is a good thing + // but for contact ID we really want to specify ID. + $this->_mapperFields['membership_contact_id'] = ts('Contact ID'); $sel1 = $this->_mapperFields; - if (!$this->get('onDuplicate')) { - unset($sel1['id']); + if (!$this->getSubmittedValue('onDuplicate')) { + // If not updating then do not allow membership id. unset($sel1['membership_id']); } @@ -163,16 +52,18 @@ public function buildQuickForm() { //used to warn for mismatch column count or mismatch mapping $warning = 0; + $savedMappingID = $this->getSubmittedValue('savedMapping'); + if ($savedMappingID) { + $fieldMappings = \Civi\Api4\MappingField::get(FALSE)->addWhere('mapping_id', '=', $savedMappingID)->execute()->indexBy('column_number'); + } - for ($i = 0; $i < $this->_columnCount; $i++) { + foreach ($columnHeaders as $i => $columnHeader) { $sel = &$this->addElement('hierselect', "mapper[$i]", ts('Mapper for Field %1', array(1 => $i)), NULL); $jsSet = FALSE; - if ($this->get('savedMapping')) { - if (isset($mappingName[$i])) { - if ($mappingName[$i] != ts('- do not import -')) { - - $mappingHeader = array_keys($this->_mapperFields, $mappingName[$i]); - + if ($this->getSubmittedValue('savedMapping')) { + $fieldMapping = $fieldMappings[$i] ?? NULL; + if (isset($fieldMappings[$i])) { + if ($fieldMapping['name'] != ts('do_not_import')) { //When locationType is not set $js .= "{$formName}['mapper[$i][1]'].style.display = 'none';\n"; @@ -181,7 +72,7 @@ public function buildQuickForm() { $js .= "{$formName}['mapper[$i][3]'].style.display = 'none';\n"; - $defaults["mapper[$i]"] = array($mappingHeader[0]); + $defaults["mapper[$i]"] = [$fieldMapping['name']]; $jsSet = TRUE; } else { @@ -198,7 +89,7 @@ public function buildQuickForm() { $js .= "swapOptions($formName, 'mapper[$i]', 0, 3, 'hs_mapper_" . $i . "_');\n"; if ($hasHeaders) { - $defaults["mapper[$i]"] = array($this->defaultFromHeader($this->_columnHeaders[$i], $headerPatterns)); + $defaults["mapper[$i]"] = array($this->defaultFromHeader($columnHeader, $headerPatterns)); } else { $defaults["mapper[$i]"] = array($this->defaultFromData($dataPatterns, $i)); @@ -208,10 +99,10 @@ public function buildQuickForm() { } else { $js .= "swapOptions($formName, 'mapper[$i]', 0, 3, 'hs_mapper_" . $i . "_');\n"; - if ($hasHeaders) { + if ($this->getSubmittedValue('skipColumnHeader')) { // Infer the default from the skipped headers if we have them $defaults["mapper[$i]"] = array( - $this->defaultFromHeader($this->_columnHeaders[$i], + $this->defaultFromHeader($columnHeader, $headerPatterns ), // $defaultLocationType->id @@ -227,7 +118,7 @@ public function buildQuickForm() { ); } } - $sel->setOptions(array($sel1, $sel2, (isset($sel3)) ? $sel3 : "", (isset($sel4)) ? $sel4 : "")); + $sel->setOptions(array($sel1, $sel2)); } $js .= "\n"; $this->assign('initHideBoxes', $js); @@ -240,11 +131,11 @@ public function buildQuickForm() { } if ($warning != 0 && $this->get('savedMapping')) { $session = CRM_Core_Session::singleton(); - $session->setStatus(ts('The data columns in this import file appear to be different from the saved mapping. Please verify that you have selected the correct saved mapping before continuing.')); + $session::setStatus(ts('The data columns in this import file appear to be different from the saved mapping. Please verify that you have selected the correct saved mapping before continuing.')); } else { $session = CRM_Core_Session::singleton(); - $session->setStatus(NULL); + $session::setStatus(NULL); } $this->setDefaults($defaults); @@ -276,7 +167,7 @@ public function buildQuickForm() { * @param $files * @param self $self * - * @return array + * @return array|bool * list of errors to be posted back to the form */ public static function formRule($fields, $files, $self) { @@ -292,18 +183,11 @@ public static function formRule($fields, $files, $self) { 'membership_type_id' => ts('Membership Type'), 'membership_start_date' => ts('Membership Start Date'), ); - - $contactTypeId = $self->get('contactType'); - $contactTypes = array( - CRM_Import_Parser::CONTACT_INDIVIDUAL => 'Individual', - CRM_Import_Parser::CONTACT_HOUSEHOLD => 'Household', - CRM_Import_Parser::CONTACT_ORGANIZATION => 'Organization', - ); $params = array( 'used' => 'Unsupervised', - 'contact_type' => $contactTypes[$contactTypeId], + 'contact_type' => $self->getContactType(), ); - list($ruleFields, $threshold) = CRM_Dedupe_BAO_DedupeRuleGroup::dedupeRuleFieldsWeight($params); + [$ruleFields, $threshold] = CRM_Dedupe_BAO_DedupeRuleGroup::dedupeRuleFieldsWeight($params); $weightSum = 0; foreach ($importKeys as $key => $val) { if (array_key_exists($val, $ruleFields)) { @@ -317,22 +201,20 @@ public static function formRule($fields, $files, $self) { foreach ($requiredFields as $field => $title) { if (!in_array($field, $importKeys)) { - if ($field == 'membership_contact_id') { + if ($field === 'membership_contact_id') { if ((($weightSum >= $threshold || in_array('external_identifier', $importKeys)) && - $self->_onDuplicate != CRM_Import_Parser::DUPLICATE_UPDATE + $self->getSubmittedValue('onDuplicate') != CRM_Import_Parser::DUPLICATE_UPDATE ) || in_array('membership_id', $importKeys) ) { continue; } - else { - if (!isset($errors['_qf_default'])) { - $errors['_qf_default'] = ''; - } - $errors['_qf_default'] .= ts('Missing required contact matching fields.') . " $fieldMessage " . ts('(Sum of all weights should be greater than or equal to threshold: %1).', array( - 1 => $threshold, - )) . ' ' . ts('(OR Membership ID if update mode.)') . '
'; + if (!isset($errors['_qf_default'])) { + $errors['_qf_default'] = ''; } + $errors['_qf_default'] .= ts('Missing required contact matching fields.') . " $fieldMessage " . ts('(Sum of all weights should be greater than or equal to threshold: %1).', array( + 1 => $threshold, + )) . ' ' . ts('(OR Membership ID if update mode.)') . '
'; } else { if (!isset($errors['_qf_default'])) { @@ -370,79 +252,12 @@ public static function formRule($fields, $files, $self) { } /** - * Process the mapped fields and map it into the uploaded file - * preview the file and extract some summary statistics + * Get the mapping name per the civicrm_mapping_field.type_id option group. * - * @return void + * @return string */ - public function postProcess() { - $params = $this->controller->exportValues('MapField'); - $this->updateUserJobMetadata('submitted_values', $this->getSubmittedValues()); - $mapper = []; - $mapperKeys = $this->controller->exportValue($this->_name, 'mapper'); - $mapperKeysMain = []; - - for ($i = 0; $i < $this->_columnCount; $i++) { - $mapper[$i] = $this->_mapperFields[$mapperKeys[$i][0]]; - $mapperKeysMain[$i] = $mapperKeys[$i][0]; - } - - $this->set('mapper', $mapper); - - // store mapping Id to display it in the preview page - if (!empty($params['mappingId'])) { - $this->set('loadMappingId', $params['mappingId']); - } - //Updating Mapping Records - if (!empty($params['updateMapping'])) { - $mappingFields = new CRM_Core_DAO_MappingField(); - $mappingFields->mapping_id = $params['mappingId']; - $mappingFields->find(); - - $mappingFieldsId = []; - while ($mappingFields->fetch()) { - if ($mappingFields->id) { - $mappingFieldsId[$mappingFields->column_number] = $mappingFields->id; - } - } - - for ($i = 0; $i < $this->_columnCount; $i++) { - $updateMappingFields = new CRM_Core_DAO_MappingField(); - $updateMappingFields->id = $mappingFieldsId[$i]; - $updateMappingFields->mapping_id = $params['mappingId']; - $updateMappingFields->column_number = $i; - $updateMappingFields->name = $mapper[$i]; - $updateMappingFields->save(); - } - } - - //Saving Mapping Details and Records - if (!empty($params['saveMapping'])) { - $mappingParams = array( - 'name' => $params['saveMappingName'], - 'description' => $params['saveMappingDesc'], - 'mapping_type_id' => CRM_Core_PseudoConstant::getKey('CRM_Core_BAO_Mapping', 'mapping_type_id', 'Import Membership'), - ); - $saveMapping = CRM_Core_BAO_Mapping::add($mappingParams); - - for ($i = 0; $i < $this->_columnCount; $i++) { - - $saveMappingFields = new CRM_Core_DAO_MappingField(); - $saveMappingFields->mapping_id = $saveMapping->id; - $saveMappingFields->column_number = $i; - $saveMappingFields->name = $mapper[$i]; - $saveMappingFields->save(); - } - $this->set('savedMapping', $saveMappingFields->mapping_id); - } - - $parser = new CRM_Member_Import_Parser_Membership($mapperKeysMain); - $parser->setUserJobID($this->getUserJobID()); - $parser->run($this->getSubmittedValue('uploadFile'), $this->getSubmittedValue('fieldSeparator'), $mapper, $this->getSubmittedValue('skipColumnHeader'), - CRM_Import_Parser::MODE_PREVIEW, $this->get('contactType') - ); - // add all the necessary variables to the form - $parser->set($this); + public function getMappingTypeName(): string { + return 'Import Membership'; } /** @@ -457,4 +272,50 @@ protected function getParser(): CRM_Member_Import_Parser_Membership { return $this->parser; } + /** + * Get the fields to be highlighted in the UI. + * + * @return array + * @throws \CRM_Core_Exception + */ + protected function getHighlightedFields(): array { + $highlightedFields = []; + //CRM-2219 removing other required fields since for update only + //membership id is required. + if ($this->getSubmittedValue('onDuplicate') == CRM_Import_Parser::DUPLICATE_UPDATE) { + $remove = [ + 'membership_contact_id', + 'email', + 'first_name', + 'last_name', + 'external_identifier', + ]; + foreach ($remove as $value) { + unset($this->_mapperFields[$value]); + } + $highlightedFieldsArray = [ + 'membership_id', + 'membership_start_date', + 'membership_type_id', + ]; + foreach ($highlightedFieldsArray as $name) { + $highlightedFields[] = $name; + } + } + elseif ($this->getSubmittedValue('onDuplicate') == CRM_Import_Parser::DUPLICATE_SKIP) { + unset($this->_mapperFields['membership_id']); + $highlightedFieldsArray = [ + 'membership_contact_id', + 'email', + 'external_identifier', + 'membership_start_date', + 'membership_type_id', + ]; + foreach ($highlightedFieldsArray as $name) { + $highlightedFields[] = $name; + } + } + return $highlightedFields; + } + } diff --git a/CRM/Member/Import/Form/Preview.php b/CRM/Member/Import/Form/Preview.php index f7d45b71d4a1..9765b7ba8d61 100644 --- a/CRM/Member/Import/Form/Preview.php +++ b/CRM/Member/Import/Form/Preview.php @@ -28,108 +28,7 @@ class CRM_Member_Import_Form_Preview extends CRM_Import_Form_Preview { */ public function preProcess() { parent::preProcess(); - //get the data from the session - $dataValues = $this->get('dataValues'); - $mapper = $this->get('mapper'); - $invalidRowCount = $this->get('invalidRowCount'); - - //get the mapping name displayed if the mappingId is set - $mappingId = $this->get('loadMappingId'); - if ($mappingId) { - $mapDAO = new CRM_Core_DAO_Mapping(); - $mapDAO->id = $mappingId; - $mapDAO->find(TRUE); - } - $this->assign('savedMappingName', $mappingId ? $mapDAO->name : NULL); - - if ($invalidRowCount) { - $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Member_Import_Parser_Membership'; - $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); - } - - $properties = [ - 'mapper', - 'dataValues', - 'columnCount', - 'totalRowCount', - 'validRowCount', - 'invalidRowCount', - 'downloadErrorRecordsUrl', - ]; $this->setStatusUrl(); - - foreach ($properties as $property) { - $this->assign($property, $this->get($property)); - } - } - - /** - * Process the mapped fields and map it into the uploaded file - * preview the file and extract some summary statistics - * - * @return void - */ - public function postProcess() { - $fileName = $this->getSubmittedValue('uploadFile'); - $invalidRowCount = $this->get('invalidRowCount'); - $onDuplicate = $this->get('onDuplicate'); - - $mapper = $this->controller->exportValue('MapField', 'mapper'); - $mapperKeys = []; - // Note: we keep the multi-dimension array (even thought it's not - // needed in the case of memberships import) so that we can merge - // the common code with contacts import later and subclass contact - // and membership imports from there - foreach ($mapper as $key => $value) { - $mapperKeys[$key] = $mapper[$key][0]; - } - - $parser = new CRM_Member_Import_Parser_Membership($mapperKeys); - $parser->setUserJobID($this->getUserJobID()); - - $mapFields = $this->get('fields'); - - foreach ($mapper as $key => $value) { - $header = []; - if (isset($mapFields[$mapper[$key][0]])) { - $header[] = $mapFields[$mapper[$key][0]]; - } - $mapperFields[] = implode(' - ', $header); - } - $parser->run($this->getSubmittedValue('uploadFile'), $this->getSubmittedValue('fieldSeparator'), - $mapperFields, - $this->getSubmittedValue('skipColumnHeader'), - CRM_Import_Parser::MODE_IMPORT, - $this->get('contactType'), - $onDuplicate, - $this->get('statusID'), - $this->get('totalRowCount') - ); - - // add all the necessary variables to the form - $parser->set($this, CRM_Import_Parser::MODE_IMPORT); - - // check if there is any error occurred - $errorStack = CRM_Core_Error::singleton(); - $errors = $errorStack->getErrors(); - $errorMessage = []; - - if (is_array($errors)) { - foreach ($errors as $key => $value) { - $errorMessage[] = $value['message']; - } - - $errorFile = $fileName['name'] . '.error.log'; - - if ($fd = fopen($errorFile, 'w')) { - fwrite($fd, implode('\n', $errorMessage)); - } - fclose($fd); - - $this->set('errorFile', $errorFile); - $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Member_Import_Parser_Membership'; - $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); - } } /** diff --git a/CRM/Member/Import/Form/Summary.php b/CRM/Member/Import/Form/Summary.php index 829b05c5fa6c..89e81e572287 100644 --- a/CRM/Member/Import/Form/Summary.php +++ b/CRM/Member/Import/Form/Summary.php @@ -20,66 +20,4 @@ */ class CRM_Member_Import_Form_Summary extends CRM_Import_Form_Summary { - /** - * Set variables up before form is built. - * - * @return void - */ - public function preProcess() { - // set the error message path to display - $this->assign('errorFile', $this->get('errorFile')); - - $totalRowCount = $this->get('totalRowCount'); - $this->set('totalRowCount', $totalRowCount); - - $invalidRowCount = $this->get('invalidRowCount'); - $duplicateRowCount = $this->get('duplicateRowCount'); - $onDuplicate = $this->get('onDuplicate'); - - if ($duplicateRowCount > 0) { - $urlParams = 'type=' . CRM_Import_Parser::DUPLICATE . '&parser=CRM_Member_Import_Parser_Membership'; - $this->set('downloadDuplicateRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); - } - else { - $duplicateRowCount = 0; - $this->set('duplicateRowCount', $duplicateRowCount); - } - - $this->assign('dupeError', FALSE); - - if ($onDuplicate == CRM_Import_Parser::DUPLICATE_UPDATE) { - $dupeActionString = ts('These records have been updated with the imported data.'); - } - elseif ($onDuplicate == CRM_Import_Parser::DUPLICATE_FILL) { - $dupeActionString = ts('These records have been filled in with the imported data.'); - } - else { - /* Skip by default */ - - $dupeActionString = ts('These records have not been imported.'); - - $this->assign('dupeError', TRUE); - - /* only subtract dupes from successful import if we're skipping */ - - $this->set('validRowCount', $totalRowCount - $invalidRowCount - - $duplicateRowCount - ); - } - $this->assign('dupeActionString', $dupeActionString); - - $properties = [ - 'totalRowCount', - 'validRowCount', - 'invalidRowCount', - 'downloadErrorRecordsUrl', - 'duplicateRowCount', - 'downloadDuplicateRecordsUrl', - 'groupAdditions', - ]; - foreach ($properties as $property) { - $this->assign($property, $this->get($property)); - } - } - } diff --git a/CRM/Member/Import/Parser/Membership.php b/CRM/Member/Import/Parser/Membership.php index 7fb151b5a98c..f3622e4cde47 100644 --- a/CRM/Member/Import/Parser/Membership.php +++ b/CRM/Member/Import/Parser/Membership.php @@ -22,9 +22,6 @@ class CRM_Member_Import_Parser_Membership extends CRM_Import_Parser { protected $_mapperKeys; - private $_membershipTypeIndex; - private $_membershipStatusIndex; - /** * Array of metadata for all available fields. * @@ -39,7 +36,6 @@ class CRM_Member_Import_Parser_Membership extends CRM_Import_Parser { */ protected $_newMemberships; - protected $_fileName; /** @@ -60,13 +56,6 @@ class CRM_Member_Import_Parser_Membership extends CRM_Import_Parser { */ protected $_lineCount; - /** - * Whether the file has a column header or not - * - * @var bool - */ - protected $_haveColumnHeader; - /** * Class constructor. * @@ -86,7 +75,6 @@ public function __construct($mapperKeys = []) { * @param int $contactType * @param int $onDuplicate * @param int $statusID - * @param int $totalRowCount * * @return mixed * @throws Exception @@ -99,166 +87,32 @@ public function run( $mode = self::MODE_PREVIEW, $contactType = self::CONTACT_INDIVIDUAL, $onDuplicate = self::DUPLICATE_SKIP, - $statusID = NULL, - $totalRowCount = NULL + $statusID = NULL ) { - if (!is_array($fileName)) { - throw new CRM_Core_Exception('Unable to determine import file'); - } - $fileName = $fileName['name']; - - switch ($contactType) { - case self::CONTACT_INDIVIDUAL: - $this->_contactType = 'Individual'; - break; - - case self::CONTACT_HOUSEHOLD: - $this->_contactType = 'Household'; - break; - - case self::CONTACT_ORGANIZATION: - $this->_contactType = 'Organization'; - } - + $this->_contactType = $this->getContactType(); $this->init(); - $this->_haveColumnHeader = $skipColumnHeader; - - $this->_separator = $separator; - - $fd = fopen($fileName, "r"); - if (!$fd) { - return FALSE; - } - $this->_lineCount = 0; $this->_invalidRowCount = $this->_validCount = 0; $this->_totalCount = 0; $this->_errors = []; $this->_warnings = []; - - $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2); - - if ($mode == self::MODE_MAPFIELD) { - $this->_rows = []; - } - else { - $this->_activeFieldCount = count($this->_activeFields); - } if ($statusID) { $this->progressImport($statusID); $startTimestamp = $currTimestamp = $prevTimestamp = CRM_Utils_Time::time(); } - - while (!feof($fd)) { - $this->_lineCount++; - - $values = fgetcsv($fd, 8192, $separator); - if (!$values) { - continue; - } - - self::encloseScrub($values); - - // skip column header if we're not in mapfield mode - if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) { - $skipColumnHeader = FALSE; - continue; - } - - /* trim whitespace around the values */ - $empty = TRUE; - foreach ($values as $k => $v) { - $values[$k] = trim($v, " \t\r\n"); - } - if (CRM_Utils_System::isNull($values)) { - continue; - } - - $this->_totalCount++; - - if ($mode == self::MODE_MAPFIELD) { - $returnCode = CRM_Import_Parser::VALID; - } - elseif ($mode == self::MODE_PREVIEW) { - $returnCode = $this->preview($values); - } - elseif ($mode == self::MODE_SUMMARY) { - $returnCode = $this->summary($values); - } - elseif ($mode == self::MODE_IMPORT) { - $returnCode = $this->import($onDuplicate, $values); + $dataSource = $this->getDataSourceObject(); + $totalRowCount = $dataSource->getRowCount(['new']); + $dataSource->setStatuses(['new']); + while ($row = $dataSource->getRow()) { + $values = array_values($row); + if ($mode == self::MODE_IMPORT) { + $this->import($values); if ($statusID && (($this->_lineCount % 50) == 0)) { $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount); } } - else { - $returnCode = self::ERROR; - } - - // note that a line could be valid but still produce a warning - if ($returnCode & self::VALID) { - $this->_validCount++; - if ($mode == self::MODE_MAPFIELD) { - $this->_rows[] = $values; - $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); - } - } - - if ($returnCode & self::ERROR) { - $this->_invalidRowCount++; - $recordNumber = $this->_lineCount; - array_unshift($values, $recordNumber); - $this->_errors[] = $values; - } - - if ($returnCode & self::DUPLICATE) { - $this->_duplicateCount++; - $recordNumber = $this->_lineCount; - array_unshift($values, $recordNumber); - $this->_duplicates[] = $values; - if ($onDuplicate != self::DUPLICATE_SKIP) { - $this->_validCount++; - } - } - - // if we are done processing the maxNumber of lines, break - if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) { - break; - } - } - - fclose($fd); - - if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) { - $customHeaders = $mapper; - - $customfields = CRM_Core_BAO_CustomField::getFields('Membership'); - foreach ($customHeaders as $key => $value) { - if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) { - $customHeaders[$key] = $customfields[$id][0]; - } - } - if ($this->_invalidRowCount) { - // removed view url for invlaid contacts - $headers = array_merge([ - ts('Line Number'), - ts('Reason'), - ], $customHeaders); - $this->_errorFileName = self::errorFileName(self::ERROR); - - self::exportCSV($this->_errorFileName, $headers, $this->_errors); - } - if ($this->_duplicateCount) { - $headers = array_merge([ - ts('Line Number'), - ts('View Membership URL'), - ], $customHeaders); - - $this->_duplicateFileName = self::errorFileName(self::DUPLICATE); - self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates); - } } } @@ -288,7 +142,8 @@ public function setActiveFields($fieldKeys) { * @return array * (reference ) associative array of name/value pairs */ - public function &getActiveFieldParams() { + public function getParams() { + $this->getSubmittedValue('mapper'); $params = []; for ($i = 0; $i < $this->_activeFieldCount; $i++) { if (isset($this->_activeFields[$i]->_value) @@ -338,38 +193,13 @@ public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPa * @return void */ public function set($store, $mode = self::MODE_SUMMARY) { - $store->set('fileSize', $this->_fileSize); $store->set('lineCount', $this->_lineCount); - $store->set('separator', $this->_separator); - $store->set('fields', $this->getSelectValues()); - - $store->set('headerPatterns', $this->getHeaderPatterns()); - $store->set('dataPatterns', $this->getDataPatterns()); - $store->set('columnCount', $this->_activeFieldCount); - - $store->set('totalRowCount', $this->_totalCount); $store->set('validRowCount', $this->_validCount); $store->set('invalidRowCount', $this->_invalidRowCount); - switch ($this->_contactType) { - case 'Individual': - $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL); - break; - - case 'Household': - $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD); - break; - - case 'Organization': - $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION); - } - if ($this->_invalidRowCount) { $store->set('errorsFileName', $this->_errorFileName); } - if (isset($this->_rows) && !empty($this->_rows)) { - $store->set('dataValues', $this->_rows); - } if ($mode == self::MODE_IMPORT) { $store->set('duplicateRowCount', $this->_duplicateCount); @@ -424,42 +254,17 @@ public static function exportCSV($fileName, $header, $data) { * @return void */ public function init() { - $this->fieldMetadata = CRM_Member_BAO_Membership::importableFields($this->_contactType, FALSE); - - foreach ($this->fieldMetadata as $name => $field) { - // @todo - we don't really need to do all this.... fieldMetadata is just fine to use as is. - $field['type'] = CRM_Utils_Array::value('type', $field, CRM_Utils_Type::T_INT); - $field['dataPattern'] = CRM_Utils_Array::value('dataPattern', $field, '//'); - $field['headerPattern'] = CRM_Utils_Array::value('headerPattern', $field, '//'); - $this->addField($name, $field['title'], $field['type'], $field['headerPattern'], $field['dataPattern']); - } + // Force re-load of user job. + unset($this->userJob); + $this->setFieldMetadata(); $this->_newMemberships = []; $this->setActiveFields($this->_mapperKeys); - - // FIXME: we should do this in one place together with Form/MapField.php - $this->_membershipTypeIndex = -1; - $this->_membershipStatusIndex = -1; - - $index = 0; - foreach ($this->_mapperKeys as $key) { - switch ($key) { - - case 'membership_type_id': - $this->_membershipTypeIndex = $index; - break; - - case 'status_id': - $this->_membershipStatusIndex = $index; - break; - } - $index++; - } } /** - * Handle the values in preview mode. + * Validate the values. * * @param array $values * The array of values belonging to this line. @@ -467,132 +272,24 @@ public function init() { * @return bool * the result of this processing */ - public function preview(&$values) { - return $this->summary($values); - } - - /** - * Handle the values in summary mode. - * - * @param array $values - * The array of values belonging to this line. - * - * @return bool - * the result of this processing - */ - public function summary(&$values) { - - $this->setActiveFieldValues($values); - - $errorRequired = FALSE; - - if ($this->_membershipTypeIndex < 0) { - $errorRequired = TRUE; - } - else { - $errorRequired = !CRM_Utils_Array::value($this->_membershipTypeIndex, $values); + public function validateValues($values) { + $params = $this->getMappedRow($values); + $errors = []; + foreach ($params as $key => $value) { + $errors = array_merge($this->getInvalidValues($value, $key), $errors); } - if ($errorRequired) { - array_unshift($values, ts('Missing required fields')); - return CRM_Import_Parser::ERROR; + if (empty($params['membership_type_id'])) { + $errors[] = ts('Missing required fields'); + return NULL; } - $params = $this->getActiveFieldParams(); - $errorMessage = NULL; - //To check whether start date or join date is provided - if (empty($params['membership_start_date']) && empty($params['membership_join_date'])) { - $errorMessage = 'Membership Start Date is required to create a memberships.'; - CRM_Contact_Import_Parser_Contact::addToErrorMsg('Start Date', $errorMessage); + if (empty($params['start_date']) && empty($params['join_date'])) { + $errors[] = 'Membership Start Date is required to create a memberships.'; } - - //for date-Formats - $session = CRM_Core_Session::singleton(); - $dateType = $session->get('dateTypes'); - foreach ($params as $key => $val) { - - if ($val) { - switch ($key) { - case 'membership_join_date': - if (CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key)) { - if (!CRM_Utils_Rule::date($params[$key])) { - CRM_Contact_Import_Parser_Contact::addToErrorMsg('Member Since', $errorMessage); - } - } - else { - CRM_Contact_Import_Parser_Contact::addToErrorMsg('Member Since', $errorMessage); - } - break; - - case 'membership_start_date': - if (CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key)) { - if (!CRM_Utils_Rule::date($params[$key])) { - CRM_Contact_Import_Parser_Contact::addToErrorMsg('Start Date', $errorMessage); - } - } - else { - CRM_Contact_Import_Parser_Contact::addToErrorMsg('Start Date', $errorMessage); - } - break; - - case 'membership_end_date': - if (CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key)) { - if (!CRM_Utils_Rule::date($params[$key])) { - CRM_Contact_Import_Parser_Contact::addToErrorMsg('End date', $errorMessage); - } - } - else { - CRM_Contact_Import_Parser_Contact::addToErrorMsg('End date', $errorMessage); - } - break; - - case 'status_override_end_date': - if (CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key)) { - if (!CRM_Utils_Rule::date($params[$key])) { - CRM_Contact_Import_Parser_Contact::addToErrorMsg('Status Override End Date', $errorMessage); - } - } - else { - CRM_Contact_Import_Parser_Contact::addToErrorMsg('Status Override End Date', $errorMessage); - } - break; - - case 'membership_type_id': - // @todo - squish into membership status - can use same lines here too. - $membershipTypes = CRM_Member_PseudoConstant::membershipType(); - if (!CRM_Utils_Array::crmInArray($val, $membershipTypes) && - !array_key_exists($val, $membershipTypes) - ) { - CRM_Contact_Import_Parser_Contact::addToErrorMsg('Membership Type', $errorMessage); - } - break; - - case 'status_id': - if (!empty($val) && !$this->parsePseudoConstantField($val, $this->fieldMetadata[$key])) { - CRM_Contact_Import_Parser_Contact::addToErrorMsg('Membership Status', $errorMessage); - } - break; - - case 'email': - if (!CRM_Utils_Rule::email($val)) { - CRM_Contact_Import_Parser_Contact::addToErrorMsg('Email Address', $errorMessage); - } - } - } - } - //date-Format part ends - - $params['contact_type'] = 'Membership'; - - //checking error in custom data - $this->isErrorInCustomData($params, $errorMessage); - - if ($errorMessage) { - $tempMsg = "Invalid value for field(s) : $errorMessage"; - array_unshift($values, $tempMsg); - $errorMessage = NULL; - return CRM_Import_Parser::ERROR; + if ($errors) { + throw new CRM_Core_Exception('Invalid value for field(s) : ' . implode(',', $errors)); } return CRM_Import_Parser::VALID; @@ -601,83 +298,26 @@ public function summary(&$values) { /** * Handle the values in import mode. * - * @param int $onDuplicate - * The code for what action to take on duplicates. * @param array $values * The array of values belonging to this line. * - * @return bool - * the result of this processing + * @return int|void|null + * the result of this processing - which is ignored */ - public function import($onDuplicate, &$values) { + public function import($values) { + $onDuplicate = $this->getSubmittedValue('onDuplicate'); + $rowNumber = (int) ($values[array_key_last($values)]); try { - // first make sure this is a valid line - $response = $this->summary($values); - if ($response != CRM_Import_Parser::VALID) { - return $response; - } - - $params = $this->getActiveFieldParams(); + $params = $this->getMappedRow($values); //assign join date equal to start date if join date is not provided - if (empty($params['membership_join_date']) && !empty($params['membership_start_date'])) { - $params['membership_join_date'] = $params['membership_start_date']; + if (empty($params['join_date']) && !empty($params['start_date'])) { + $params['join_date'] = $params['start_date']; } - $session = CRM_Core_Session::singleton(); - $dateType = CRM_Core_Session::singleton()->get('dateTypes'); - $formatted = []; - $customDataType = !empty($params['contact_type']) ? $params['contact_type'] : 'Membership'; - $customFields = CRM_Core_BAO_CustomField::getFields($customDataType); - + $formatted = $params; // don't add to recent items, CRM-4399 $formatted['skipRecentView'] = TRUE; - $dateLabels = [ - 'membership_join_date' => ts('Member Since'), - 'membership_start_date' => ts('Start Date'), - 'membership_end_date' => ts('End Date'), - ]; - foreach ($params as $key => $val) { - if ($val) { - switch ($key) { - case 'membership_join_date': - case 'membership_start_date': - case 'membership_end_date': - if (CRM_Utils_Date::convertToDefaultDate($params, $dateType, $key)) { - if (!CRM_Utils_Rule::date($params[$key])) { - CRM_Contact_Import_Parser_Contact::addToErrorMsg($dateLabels[$key], $errorMessage); - } - } - else { - CRM_Contact_Import_Parser_Contact::addToErrorMsg($dateLabels[$key], $errorMessage); - } - break; - - case 'membership_type_id': - if (!is_numeric($val)) { - unset($params['membership_type_id']); - $params['membership_type'] = $val; - } - break; - - case 'status_id': - // @todo - we can do this based on the presence of 'pseudoconstant' in the metadata rather than field specific. - $params[$key] = $this->parsePseudoConstantField($val, $this->fieldMetadata[$key]); - break; - - } - if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) { - if ($customFields[$customFieldID]['data_type'] == 'Date') { - $this->formatCustomDate($params, $formatted, $dateType, $key); - unset($params[$key]); - } - elseif ($customFields[$customFieldID]['data_type'] == 'Boolean') { - $params[$key] = CRM_Utils_String::strtoboolstr($val); - } - } - } - } - //date-Format part ends $formatValues = []; foreach ($params as $key => $field) { @@ -702,9 +342,8 @@ public function import($onDuplicate, &$values) { else { //fix for CRM-2219 Update Membership // onDuplicate == CRM_Import_Parser::DUPLICATE_UPDATE - if (!empty($formatted['member_is_override']) && empty($formatted['status_id'])) { - array_unshift($values, 'Required parameter missing: Status'); - return CRM_Import_Parser::ERROR; + if (!empty($formatted['is_override']) && empty($formatted['status_id'])) { + throw new CRM_Core_Exception('Required parameter missing: Status', CRM_Import_Parser::ERROR); } if (!empty($formatValues['membership_id'])) { @@ -728,28 +367,25 @@ public function import($onDuplicate, &$values) { $newMembership = civicrm_api3('Membership', 'create', $formatted); $this->_newMemberships[] = $newMembership['id']; + $this->setImportStatus($rowNumber, 'IMPORTED', 'Required parameter missing: Status'); return CRM_Import_Parser::VALID; } - else { - array_unshift($values, 'Matching Membership record not found for Membership ID ' . $formatValues['membership_id'] . '. Row was skipped.'); - return CRM_Import_Parser::ERROR; - } + throw new CRM_Core_Exception('Matching Membership record not found for Membership ID ' . $formatValues['membership_id'] . '. Row was skipped.', CRM_Import_Parser::ERROR); } } //Format dates - $startDate = CRM_Utils_Date::customFormat(CRM_Utils_Array::value('start_date', $formatted), '%Y-%m-%d'); - $endDate = CRM_Utils_Date::customFormat(CRM_Utils_Array::value('end_date', $formatted), '%Y-%m-%d'); - $joinDate = CRM_Utils_Date::customFormat(CRM_Utils_Array::value('join_date', $formatted), '%Y-%m-%d'); + $startDate = $formatted['start_date']; + $endDate = $formatted['end_date'] ?? NULL; + $joinDate = $formatted['join_date']; - if (!$this->isContactIDColumnPresent()) { + if (empty($formatValues['id']) && empty($formatValues['contact_id'])) { $error = $this->checkContactDuplicate($formatValues); if (CRM_Core_Error::isAPIError($error, CRM_Core_ERROR::DUPLICATE_CONTACT)) { $matchedIDs = explode(',', $error['error_message']['params'][0]); if (count($matchedIDs) > 1) { - array_unshift($values, 'Multiple matching contact records detected for this row. The membership was not imported'); - return CRM_Import_Parser::ERROR; + throw new CRM_Core_Exception('Multiple matching contact records detected for this row. The membership was not imported', CRM_Import_Parser::ERROR); } else { $cid = $matchedIDs[0]; @@ -766,7 +402,7 @@ public function import($onDuplicate, &$values) { //fix for CRM-3570, exclude the statuses those having is_admin = 1 //now user can import is_admin if is override is true. $excludeIsAdmin = FALSE; - if (empty($formatted['member_is_override'])) { + if (empty($formatted['is_override'])) { $formatted['exclude_is_admin'] = $excludeIsAdmin = TRUE; } $calcStatus = CRM_Member_BAO_MembershipStatus::getMembershipStatusByDate($startDate, @@ -781,21 +417,20 @@ public function import($onDuplicate, &$values) { if (empty($formatted['status_id'])) { $formatted['status_id'] = $calcStatus['id']; } - elseif (empty($formatted['member_is_override'])) { + elseif (empty($formatted['is_override'])) { if (empty($calcStatus)) { - array_unshift($values, 'Status in import row (' . $formatValues['status_id'] . ') does not match calculated status based on your configured Membership Status Rules. Record was not imported.'); - return CRM_Import_Parser::ERROR; + throw new CRM_Core_Exception('Status in import row (' . $formatValues['status_id'] . ') does not match calculated status based on your configured Membership Status Rules. Record was not imported.', CRM_Import_Parser::ERROR); } - elseif ($formatted['status_id'] != $calcStatus['id']) { + if ($formatted['status_id'] != $calcStatus['id']) { //Status Hold" is either NOT mapped or is FALSE - array_unshift($values, 'Status in import row (' . $formatValues['status_id'] . ') does not match calculated status based on your configured Membership Status Rules (' . $calcStatus['name'] . '). Record was not imported.'); - return CRM_Import_Parser::ERROR; + throw new CRM_Core_Exception('Status in import row (' . $formatValues['status_id'] . ') does not match calculated status based on your configured Membership Status Rules (' . $calcStatus['name'] . '). Record was not imported.', CRM_Import_Parser::ERROR); } } $newMembership = civicrm_api3('membership', 'create', $formatted); $this->_newMemberships[] = $newMembership['id']; + $this->setImportStatus($rowNumber, 'IMPORTED', ''); return CRM_Import_Parser::VALID; } } @@ -828,9 +463,7 @@ public function import($onDuplicate, &$values) { $disp = $params['external_identifier']; } } - - array_unshift($values, 'No matching Contact found for (' . $disp . ')'); - return CRM_Import_Parser::ERROR; + throw new CRM_Core_Exception('No matching Contact found for (' . $disp . ')', CRM_Import_Parser::ERROR); } } else { @@ -839,8 +472,7 @@ public function import($onDuplicate, &$values) { $checkCid->external_identifier = $formatValues['external_identifier']; $checkCid->find(TRUE); if ($checkCid->id != $formatted['contact_id']) { - array_unshift($values, 'Mismatch of External ID:' . $formatValues['external_identifier'] . ' and Contact Id:' . $formatted['contact_id']); - return CRM_Import_Parser::ERROR; + throw new CRM_Core_Exception('Mismatch of External ID:' . $formatValues['external_identifier'] . ' and Contact Id:' . $formatted['contact_id'], CRM_Import_Parser::ERROR); } } @@ -856,7 +488,7 @@ public function import($onDuplicate, &$values) { //fix for CRM-3570, exclude the statuses those having is_admin = 1 //now user can import is_admin if is override is true. $excludeIsAdmin = FALSE; - if (empty($formatted['member_is_override'])) { + if (empty($formatted['is_override'])) { $formatted['exclude_is_admin'] = $excludeIsAdmin = TRUE; } $calcStatus = CRM_Member_BAO_MembershipStatus::getMembershipStatusByDate($startDate, @@ -870,26 +502,29 @@ public function import($onDuplicate, &$values) { if (empty($formatted['status_id'])) { $formatted['status_id'] = $calcStatus['id'] ?? NULL; } - elseif (empty($formatted['member_is_override'])) { + elseif (empty($formatted['is_override'])) { if (empty($calcStatus)) { - array_unshift($values, 'Status in import row (' . CRM_Utils_Array::value('status_id', $formatValues) . ') does not match calculated status based on your configured Membership Status Rules. Record was not imported.'); - return CRM_Import_Parser::ERROR; + throw new CRM_Core_Exception('Status in import row (' . CRM_Utils_Array::value('status_id', $formatValues) . ') does not match calculated status based on your configured Membership Status Rules. Record was not imported.', CRM_Import_Parser::ERROR); } - elseif ($formatted['status_id'] != $calcStatus['id']) { + if ($formatted['status_id'] != $calcStatus['id']) { //Status Hold" is either NOT mapped or is FALSE - array_unshift($values, 'Status in import row (' . CRM_Utils_Array::value('status_id', $formatValues) . ') does not match calculated status based on your configured Membership Status Rules (' . $calcStatus['name'] . '). Record was not imported.'); - return CRM_Import_Parser::ERROR; + throw new CRM_Core_Exception($rowNumber, 'ERROR', 'Status in import row (' . CRM_Utils_Array::value('status_id', $formatValues) . ') does not match calculated status based on your configured Membership Status Rules (' . $calcStatus['name'] . '). Record was not imported.', CRM_Import_Parser::ERROR); } } $newMembership = civicrm_api3('membership', 'create', $formatted); $this->_newMemberships[] = $newMembership['id']; + $this->setImportStatus($rowNumber, 'IMPORTED', ''); return CRM_Import_Parser::VALID; } } - catch (Exception $e) { - array_unshift($values, $e->getMessage()); + catch (CRM_Core_Exception $e) { + $this->setImportStatus($rowNumber, 'ERROR', $e->getMessage()); + return CRM_Import_Parser::ERROR; + } + catch (CiviCRM_API3_Exception $e) { + $this->setImportStatus($rowNumber, 'ERROR', $e->getMessage()); return CRM_Import_Parser::ERROR; } } @@ -958,17 +593,8 @@ public function membership_format_params($params, &$values, $create = FALSE) { foreach ($params as $key => $value) { - //Handling Custom Data - if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) { - $values[$key] = $value; - $type = $customFields[$customFieldID]['html_type']; - if (CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID])) { - $values[$key] = self::unserializeCustomValue($customFieldID, $value, $type); - } - } - switch ($key) { - case 'membership_contact_id': + case 'contact_id': if (!CRM_Utils_Rule::integer($value)) { throw new Exception("contact_id not valid: $value"); } @@ -980,69 +606,49 @@ public function membership_format_params($params, &$values, $create = FALSE) { if (!$svq) { throw new Exception("Invalid Contact ID: There is no contact record with contact_id = $value."); } - $values['contact_id'] = $values['membership_contact_id']; - unset($values['membership_contact_id']); - break; - - case 'membership_type_id': - if (!array_key_exists($value, CRM_Member_PseudoConstant::membershipType())) { - throw new Exception('Invalid Membership Type Id'); - } $values[$key] = $value; break; - case 'membership_type': - $membershipTypeId = CRM_Utils_Array::key(ucfirst($value), - CRM_Member_PseudoConstant::membershipType() - ); - if ($membershipTypeId) { - if (!empty($values['membership_type_id']) && - $membershipTypeId != $values['membership_type_id'] - ) { - throw new Exception('Mismatched membership Type and Membership Type Id'); - } - } - else { - throw new Exception('Invalid Membership Type'); - } - $values['membership_type_id'] = $membershipTypeId; - break; - default: break; } } - if ($create) { - // CRM_Member_BAO_Membership::create() handles membership_start_date, membership_join_date, - // membership_end_date and membership_source. So, if $values contains - // membership_start_date, membership_end_date, membership_join_date or membership_source, - // convert it to start_date, end_date, join_date or source - $changes = [ - 'membership_join_date' => 'join_date', - 'membership_start_date' => 'start_date', - 'membership_end_date' => 'end_date', - 'membership_source' => 'source', - ]; - - foreach ($changes as $orgVal => $changeVal) { - if (isset($values[$orgVal])) { - $values[$changeVal] = $values[$orgVal]; - unset($values[$orgVal]); - } - } - } - return NULL; } /** - * Is the contact ID mapped. + * Set field metadata. + */ + protected function setFieldMetadata(): void { + if (empty($this->importableFieldsMetadata)) { + $metadata = CRM_Member_BAO_Membership::importableFields($this->getContactType(), FALSE); + + foreach ($metadata as $name => $field) { + // @todo - we don't really need to do all this.... fieldMetadata is just fine to use as is. + $field['type'] = CRM_Utils_Array::value('type', $field, CRM_Utils_Type::T_INT); + $field['dataPattern'] = CRM_Utils_Array::value('dataPattern', $field, '//'); + $field['headerPattern'] = CRM_Utils_Array::value('headerPattern', $field, '//'); + $this->addField($name, $field['title'], $field['type'], $field['headerPattern'], $field['dataPattern']); + } + // We are consolidating on `importableFieldsMetadata` - but both still used. + $this->importableFieldsMetadata = $this->fieldMetadata = $metadata; + } + } + + /** + * Get the metadata field for which importable fields does not key the actual field name. * - * @return bool + * @return string[] */ - protected function isContactIDColumnPresent(): bool { - return in_array('membership_contact_id', $this->_mapperKeys, TRUE); + protected function getOddlyMappedMetadataFields(): array { + $uniqueNames = ['membership_id', 'membership_contact_id']; + $fields = []; + foreach ($uniqueNames as $name) { + $fields[$this->importableFieldsMetadata[$name]['name']] = $name; + } + // Include the parent fields as they could be present if required for matching ...in theory. + return array_merge($fields, parent::getOddlyMappedMetadataFields()); } } diff --git a/CRM/Upgrade/Incremental/php/FiveFiftyOne.php b/CRM/Upgrade/Incremental/php/FiveFiftyOne.php index 3de256e0ff4b..f3a8f2dff409 100644 --- a/CRM/Upgrade/Incremental/php/FiveFiftyOne.php +++ b/CRM/Upgrade/Incremental/php/FiveFiftyOne.php @@ -91,6 +91,7 @@ public static function convertMappingFieldLabelsToNames(): bool { $fieldMap[ts(ts('Payment Method'))] = 'payment_instrument_id'; $fieldMap[ts('- do not import -')] = 'do_not_import'; + // Membership fields foreach ($mappings as $mapping) { if (!empty($fieldMap[$mapping['name']])) { MappingField::update(FALSE) @@ -99,6 +100,32 @@ public static function convertMappingFieldLabelsToNames(): bool { ->execute(); } } + + // Membership fields... + // Yes - I know they could be combined - but it's also less confusing this way. + $mappings = MappingField::get(FALSE) + ->setSelect(['id', 'name']) + ->addWhere('mapping_id.mapping_type_id:name', '=', 'Import Membership') + ->execute(); + $fields = CRM_Member_BAO_Membership::importableFields('All', FALSE);; + $fieldMap = []; + foreach ($fields as $fieldName => $field) { + $fieldMap[$field['title']] = $fieldName; + if (!empty($field['html']['label'])) { + $fieldMap[$field['html']['label']] = $fieldName; + } + } + $fieldMap[ts('- do not import -')] = 'do_not_import'; + + foreach ($mappings as $mapping) { + if (!empty($fieldMap[$mapping['name']])) { + MappingField::update(FALSE) + ->addWhere('id', '=', $mapping['id']) + ->addValue('name', $fieldMap[$mapping['name']]) + ->execute(); + } + } + return TRUE; } diff --git a/templates/CRM/Member/Import/Form/MapField.tpl b/templates/CRM/Member/Import/Form/MapField.tpl index 691755c7fb10..287e40ba0dfe 100644 --- a/templates/CRM/Member/Import/Form/MapField.tpl +++ b/templates/CRM/Member/Import/Form/MapField.tpl @@ -19,7 +19,7 @@ {* @var $form Contains the array for the form elements and other form associated information assigned to the template by the controller *} {* Table for mapping data to CRM fields *} - {include file="CRM/Import/Form/MapTable.tpl"} + {include file="CRM/Import/Form/MapTableCommon.tpl" mapper=$form.mapper}
{include file="CRM/common/formButtons.tpl" location="bottom"}
diff --git a/templates/CRM/Member/Import/Form/Preview.tpl b/templates/CRM/Member/Import/Form/Preview.tpl index 809193341af1..aa28d8b6c031 100644 --- a/templates/CRM/Member/Import/Form/Preview.tpl +++ b/templates/CRM/Member/Import/Form/Preview.tpl @@ -55,7 +55,7 @@ {* Table for mapping preview *} - {include file="CRM/Import/Form/MapTable.tpl"} + {include file="CRM/Import/Form/MapTableCommon.tpl"}
{include file="CRM/common/formButtons.tpl" location="bottom"}
diff --git a/tests/phpunit/CRM/Member/Import/Parser/MembershipTest.php b/tests/phpunit/CRM/Member/Import/Parser/MembershipTest.php index 4c2ff9bfc3fa..212ed0e6fe43 100644 --- a/tests/phpunit/CRM/Member/Import/Parser/MembershipTest.php +++ b/tests/phpunit/CRM/Member/Import/Parser/MembershipTest.php @@ -1,7 +1,7 @@ . */ +use Civi\Api4\UserJob; + /** - * Test CRM/Member/BAO Membership Log add , delete functions - * * @package CiviCRM * @group headless */ class CRM_Member_Import_Parser_MembershipTest extends CiviUnitTestCase { use CRMTraits_Custom_CustomDataTrait; + /** + * @var int + */ + protected $userJobID; + /** * Membership type name used in test function. * @@ -45,7 +50,9 @@ class CRM_Member_Import_Parser_MembershipTest extends CiviUnitTestCase { * * @var string */ - protected $_membershipTypeID = NULL; + protected $_membershipTypeID; + + protected $entity = 'Membership'; /** * Set up for test. @@ -65,6 +72,7 @@ public function setUp(): void { $this->_relationshipTypeId = $this->relationshipTypeCreate($params); $this->_orgContactID = $this->organizationCreate(); $this->_financialTypeId = 1; + $this->restoreMembershipTypes(); $this->_membershipTypeName = 'Mickey Mouse Club Member'; $params = [ 'name' => $this->_membershipTypeName, @@ -86,8 +94,6 @@ public function setUp(): void { $this->_membershipTypeID = $membershipType->id; $this->_mebershipStatusID = $this->membershipStatusCreate('test status'); - $session = CRM_Core_Session::singleton(); - $session->set('dateTypes', 1); } /** @@ -103,6 +109,10 @@ public function tearDown(): void { 'civicrm_contribution', 'civicrm_membership_payment', 'civicrm_contact', + 'civicrm_email', + 'civicrm_user_job', + 'civicrm_queue', + 'civicrm_queue_item', ]; $this->relationshipTypeDelete($this->_relationshipTypeId); $this->membershipTypeDelete(['id' => $this->_membershipTypeID]); @@ -147,7 +157,7 @@ public function testImport(): void { $importObject = $this->createImportObject(['email', 'membership_type_id', 'membership_start_date', 'membership_join_date']); foreach ($params as $values) { - $this->assertEquals(CRM_Import_Parser::VALID, $importObject->import(CRM_Import_Parser::DUPLICATE_UPDATE, $values), $values[0]); + $this->assertEquals(CRM_Import_Parser::VALID, $importObject->import($values), $values[0]); } $result = $this->callAPISuccess('membership', 'get', ['sequential' => 1])['values']; $this->assertCount(2, $result); @@ -160,16 +170,13 @@ public function testImport(): void { * * @throws \CRM_Core_Exception */ - public function testImportOverriddenMembershipButWithoutStatus() { + public function testImportOverriddenMembershipButWithoutStatus(): void { $this->individualCreate(['email' => 'anthony_anderson2@civicrm.org']); - - $fieldMapper = [ - 'mapper[0][0]' => 'email', - 'mapper[1][0]' => 'membership_type_id', - 'mapper[2][0]' => 'membership_start_date', - 'mapper[3][0]' => 'member_is_override', - ]; - $membershipImporter = new CRM_Member_Import_Parser_Membership($fieldMapper); + $membershipImporter = new CRM_Member_Import_Parser_Membership(); + $membershipImporter->setUserJobID($this->getUserJobID([ + 'mapper' => [['email'], ['membership_type_id'], ['membership_start_date'], ['member_is_override']], + 'onDuplicate' => CRM_Import_Parser::DUPLICATE_UPDATE, + ])); $membershipImporter->init(); $membershipImporter->_contactType = 'Individual'; @@ -180,7 +187,7 @@ public function testImportOverriddenMembershipButWithoutStatus() { TRUE, ]; - $importResponse = $membershipImporter->import(CRM_Import_Parser::DUPLICATE_UPDATE, $importValues); + $importResponse = $membershipImporter->import($importValues); $this->assertEquals(CRM_Import_Parser::ERROR, $importResponse); $this->assertContains('Required parameter missing: Status', $importValues); } @@ -208,24 +215,17 @@ public function testImportOverriddenMembershipWithStatus() { 'New', ]; - $importResponse = $membershipImporter->import(CRM_Import_Parser::DUPLICATE_UPDATE, $importValues); + $importResponse = $membershipImporter->import($importValues); $this->assertEquals(CRM_Import_Parser::VALID, $importResponse); } - public function testImportOverriddenMembershipWithValidOverrideEndDate() { + public function testImportOverriddenMembershipWithValidOverrideEndDate(): void { $this->individualCreate(['email' => 'anthony_anderson4@civicrm.org']); - - $fieldMapper = [ - 'mapper[0][0]' => 'email', - 'mapper[1][0]' => 'membership_type_id', - 'mapper[2][0]' => 'membership_start_date', - 'mapper[3][0]' => 'member_is_override', - 'mapper[4][0]' => 'status_id', - 'mapper[5][0]' => 'status_override_end_date', - ]; - $membershipImporter = new CRM_Member_Import_Parser_Membership($fieldMapper); + $membershipImporter = new CRM_Member_Import_Parser_Membership(); + $membershipImporter->setUserJobID($this->getUserJobID([ + 'mapper' => [['email'], ['membership_type_id'], ['membership_start_date'], ['member_is_override'], ['status_id'], ['status_override_end_date']], + ])); $membershipImporter->init(); - $membershipImporter->_contactType = 'Individual'; $importValues = [ 'anthony_anderson4@civicrm.org', @@ -236,37 +236,37 @@ public function testImportOverriddenMembershipWithValidOverrideEndDate() { date('Y-m-d'), ]; - $importResponse = $membershipImporter->import(CRM_Import_Parser::DUPLICATE_UPDATE, $importValues); + $importResponse = $membershipImporter->import($importValues); $this->assertEquals(CRM_Import_Parser::VALID, $importResponse); } - public function testImportOverriddenMembershipWithInvalidOverrideEndDate() { + public function testImportOverriddenMembershipWithInvalidOverrideEndDate(): void { $this->individualCreate(['email' => 'anthony_anderson5@civicrm.org']); - - $fieldMapper = [ - 'mapper[0][0]' => 'email', - 'mapper[1][0]' => 'membership_type_id', - 'mapper[2][0]' => 'membership_start_date', - 'mapper[3][0]' => 'member_is_override', - 'mapper[4][0]' => 'status_id', - 'mapper[5][0]' => 'status_override_end_date', - ]; - $membershipImporter = new CRM_Member_Import_Parser_Membership($fieldMapper); + $this->userJobID = $this->getUserJobID([ + 'mapper' => [['email'], ['membership_type_id'], ['membership_start_date'], ['member_is_override'], ['status_id'], ['status_override_end_date']], + 'onDuplicate' => CRM_Import_Parser::DUPLICATE_UPDATE, + ]); + $membershipImporter = new CRM_Member_Import_Parser_Membership(); + $membershipImporter->setUserJobID($this->userJobID); $membershipImporter->init(); - $membershipImporter->_contactType = 'Individual'; $importValues = [ 'anthony_anderson5@civicrm.org', - 'New', + 'General', date('Y-m-d'), - TRUE, + 1, $this->_mebershipStatusID, 'abc', ]; + try { + $importResponse = $membershipImporter->validateValues($importValues); + } + catch (CRM_Core_Exception $e) { + $this->assertEquals('Invalid value for field(s) : Status Override End Date', $e->getMessage()); + return; + } + $this->fail('Exception expected'); - $importResponse = $membershipImporter->import(CRM_Import_Parser::DUPLICATE_UPDATE, $importValues); - $this->assertEquals(CRM_Import_Parser::ERROR, $importResponse); - $this->assertContains('Required parameter missing: Status', $importValues); } /** @@ -299,7 +299,7 @@ public function testImportMembershipWithRenamedStatus() { 'New-renamed', ]; - $importResponse = $membershipImporter->import(CRM_Import_Parser::DUPLICATE_UPDATE, $importValues); + $importResponse = $membershipImporter->import($importValues); $this->assertEquals(CRM_Import_Parser::VALID, $importResponse); $createdStatusID = $this->callAPISuccessGetValue('Membership', ['return' => 'status_id']); $this->assertEquals(CRM_Core_PseudoConstant::getKey('CRM_Member_BAO_Membership', 'status_id', 'New'), $createdStatusID); @@ -320,15 +320,50 @@ public function testImportMembershipWithRenamedStatus() { */ protected function createImportObject(array $fields): \CRM_Member_Import_Parser_Membership { $fieldMapper = []; + $mapper = []; foreach ($fields as $index => $field) { $fieldMapper['mapper[' . $index . '][0]'] = $field; + $mapper[] = [$field]; } + $membershipImporter = new CRM_Member_Import_Parser_Membership($fieldMapper); + $membershipImporter->setUserJobID($this->getUserJobID(['mapper' => $mapper])); $membershipImporter->init(); $membershipImporter->_contactType = 'Individual'; return $membershipImporter; } + /** + * @param array $submittedValues + * + * @return int + */ + protected function getUserJobID(array $submittedValues = []): int { + $userJobID = UserJob::create()->setValues([ + 'metadata' => [ + 'submitted_values' => array_merge([ + 'contactType' => CRM_Import_Parser::CONTACT_INDIVIDUAL, + 'contactSubType' => '', + 'dataSource' => 'CRM_Import_DataSource_SQL', + 'sqlQuery' => 'SELECT first_name FROM civicrm_contact', + 'onDuplicate' => CRM_Import_Parser::DUPLICATE_SKIP, + 'dedupe_rule_id' => NULL, + 'dateFormats' => CRM_Core_Form_Date::DATE_yyyy_mm_dd, + ], $submittedValues), + ], + 'status_id:name' => 'draft', + 'type_id:name' => 'contact_import', + ])->execute()->first()['id']; + if ($submittedValues['dataSource'] ?? NULL === 'CRM_Import_DataSource') { + $dataSource = new CRM_Import_DataSource_CSV($userJobID); + } + else { + $dataSource = new CRM_Import_DataSource_SQL($userJobID); + } + $dataSource->initialize(); + return $userJobID; + } + /** * Test importing to a custom field. * @@ -353,11 +388,125 @@ public function testImportCustomData(): void { 'Red', ]; - $importResponse = $membershipImporter->import(CRM_Import_Parser::DUPLICATE_UPDATE, $importValues); + $importResponse = $membershipImporter->import($importValues); $this->assertEquals(CRM_Import_Parser::VALID, $importResponse); $membership = $this->callAPISuccessGetSingle('Membership', []); $this->assertEquals('blah', $membership[$this->getCustomFieldName('text')]); $this->assertEquals('R', $membership[$this->getCustomFieldName('select_string')]); } + /** + * Test the full form-flow import. + */ + public function testImportCSV() :void { + $this->importCSV('memberships_invalid.csv', [ + ['name' => 'membership_contact_id'], + ['name' => 'membership_source'], + ['name' => 'membership_type_id'], + ['name' => 'membership_start_date'], + ['name' => 'do_not_import'], + ]); + $dataSource = new CRM_Import_DataSource_CSV($this->userJobID); + $row = $dataSource->getRow(); + $this->assertEquals('ERROR', $row['_status']); + $this->assertEquals('Invalid value for field(s) : Membership Type', $row['_status_message']); + } + + /** + * Test the full form-flow import. + */ + public function testImportTSV() :void { + $this->individualCreate(['email' => 'member@example.com']); + $this->importCSV('memberships_valid.tsv', [ + ['name' => 'email'], + ['name' => 'membership_source'], + ['name' => 'membership_type_id'], + ['name' => 'membership_start_date'], + ['name' => 'do_not_import'], + ], ['fieldSeparator' => 'tab']); + $dataSource = new CRM_Import_DataSource_CSV($this->userJobID); + $row = $dataSource->getRow(); + $this->assertEquals('IMPORTED', $row['_status']); + $this->callAPISuccessGetSingle('Membership', []); + } + + /** + * Test dates are parsed. + */ + public function testUpdateWithCustomDates(): void { + $this->createCustomGroupWithFieldOfType([], 'date'); + $contactID = $this->individualCreate(['external_identifier' => 'ext-1']); + $this->callAPISuccess('Membership', 'create', [ + 'contact_id' => $contactID, + 'membership_type_id' => 'General', + 'start_date' => '2020-10-01', + ]); + $mapping = [ + ['name' => 'membership_id'], + ['name' => 'membership_source'], + ['name' => 'membership_type_id'], + ['name' => 'membership_start_date'], + ['name' => $this->getCustomFieldName('date')], + ]; + $this->importCSV('memberships_update_custom_date.csv', $mapping, ['dateFormats' => 32]); + $membership = $this->callAPISuccessGetSingle('Membership', []); + $this->assertEquals('2021-03-23', $membership['start_date']); + $this->assertEquals('2019-03-23 00:00:00', $membership[$this->getCustomFieldName('date')]); + } + + /** + * Import the csv file values. + * + * This function uses a flow that mimics the UI flow. + * + * @param string $csv Name of csv file. + * @param array $fieldMappings + * @param array $submittedValues + */ + protected function importCSV(string $csv, array $fieldMappings, array $submittedValues = []): void { + $submittedValues = array_merge([ + 'uploadFile' => ['name' => __DIR__ . '/data/' . $csv], + 'skipColumnHeader' => TRUE, + 'fieldSeparator' => ',', + 'contactType' => CRM_Import_Parser::CONTACT_INDIVIDUAL, + 'mapper' => $this->getMapperFromFieldMappings($fieldMappings), + 'dataSource' => 'CRM_Import_DataSource_CSV', + 'file' => ['name' => $csv], + 'dateFormats' => CRM_Core_Form_Date::DATE_yyyy_mm_dd, + 'onDuplicate' => CRM_Import_Parser::DUPLICATE_UPDATE, + 'groups' => [], + ], $submittedValues); + $form = $this->getFormObject('CRM_Member_Import_Form_DataSource', $submittedValues); + $values = $_SESSION['_' . $form->controller->_name . '_container']['values']; + $form->buildForm(); + $form->postProcess(); + // This gets reset in DataSource so re-do.... + $_SESSION['_' . $form->controller->_name . '_container']['values'] = $values; + + $this->userJobID = $form->getUserJobID(); + $form = $this->getFormObject('CRM_Member_Import_Form_MapField', $submittedValues); + $form->setUserJobID($this->userJobID); + $form->buildForm(); + $form->postProcess(); + /* @var CRM_Member_Import_Form_MapField $form */ + $form = $this->getFormObject('CRM_Member_Import_Form_Preview', $submittedValues); + $form->setUserJobID($this->userJobID); + $form->buildForm(); + $form->postProcess(); + } + + /** + * @param array $mappings + * + * @return array + */ + protected function getMapperFromFieldMappings(array $mappings): array { + $mapper = []; + foreach ($mappings as $mapping) { + $fieldInput = [$mapping['name']]; + $mapper[] = $fieldInput; + } + return $mapper; + } + } diff --git a/tests/phpunit/CRM/Member/Import/Parser/data/memberships.csv b/tests/phpunit/CRM/Member/Import/Parser/data/memberships.csv new file mode 100644 index 000000000000..56acafa69375 --- /dev/null +++ b/tests/phpunit/CRM/Member/Import/Parser/data/memberships.csv @@ -0,0 +1,2 @@ +Contact ID,Source,Membership Type,Start Date,Ignore me +1,Import,Made Up,2019-03-23,Just some cruft diff --git a/tests/phpunit/CRM/Member/Import/Parser/data/memberships_invalid.csv b/tests/phpunit/CRM/Member/Import/Parser/data/memberships_invalid.csv new file mode 100644 index 000000000000..56acafa69375 --- /dev/null +++ b/tests/phpunit/CRM/Member/Import/Parser/data/memberships_invalid.csv @@ -0,0 +1,2 @@ +Contact ID,Source,Membership Type,Start Date,Ignore me +1,Import,Made Up,2019-03-23,Just some cruft diff --git a/tests/phpunit/CRM/Member/Import/Parser/data/memberships_update_custom_date.csv b/tests/phpunit/CRM/Member/Import/Parser/data/memberships_update_custom_date.csv new file mode 100644 index 000000000000..f49273cef358 --- /dev/null +++ b/tests/phpunit/CRM/Member/Import/Parser/data/memberships_update_custom_date.csv @@ -0,0 +1,2 @@ +Membership ID,Source,Membership Type,Start Date,Custom Date +1,Import,General,23/03/2021,23/03/2019 diff --git a/tests/phpunit/CRM/Member/Import/Parser/data/memberships_valid.csv b/tests/phpunit/CRM/Member/Import/Parser/data/memberships_valid.csv new file mode 100644 index 000000000000..86c7037dacf6 --- /dev/null +++ b/tests/phpunit/CRM/Member/Import/Parser/data/memberships_valid.csv @@ -0,0 +1,2 @@ +email,Source,Membership Type,Start Date,Ignore me +member@example.com,Import,General,2019-03-23,Just some cruft diff --git a/tests/phpunit/CRM/Member/Import/Parser/data/memberships_valid.tsv b/tests/phpunit/CRM/Member/Import/Parser/data/memberships_valid.tsv new file mode 100644 index 000000000000..925aec034b42 --- /dev/null +++ b/tests/phpunit/CRM/Member/Import/Parser/data/memberships_valid.tsv @@ -0,0 +1,2 @@ +email Source Membership Type Start Date Ignore me +member@example.com Import General 2019-03-23 Just some cruft diff --git a/tests/phpunit/CiviTest/CiviUnitTestCase.php b/tests/phpunit/CiviTest/CiviUnitTestCase.php index 2bbbd7d67e66..d0959aba7d6e 100644 --- a/tests/phpunit/CiviTest/CiviUnitTestCase.php +++ b/tests/phpunit/CiviTest/CiviUnitTestCase.php @@ -3243,6 +3243,18 @@ public function getFormObject($class, $formValues = [], $pageName = '', $searchF $_SESSION['_' . $form->controller->_name . '_container']['values']['Preview'] = $formValues; return $form; + case 'CRM_Member_Import_Form_DataSource': + case 'CRM_Member_Import_Form_MapField': + case 'CRM_Member_Import_Form_Preview': + $form->controller = new CRM_Member_Import_Controller(); + $form->controller->setStateMachine(new CRM_Core_StateMachine($form->controller)); + // The submitted values should be set on one or the other of the forms in the flow. + // For test simplicity we set on all rather than figuring out which ones go where.... + $_SESSION['_' . $form->controller->_name . '_container']['values']['DataSource'] = $formValues; + $_SESSION['_' . $form->controller->_name . '_container']['values']['MapField'] = $formValues; + $_SESSION['_' . $form->controller->_name . '_container']['values']['Preview'] = $formValues; + return $form; + case strpos($class, '_Form_') !== FALSE: $form->controller = new CRM_Core_Controller_Simple($class, $pageName); break;