Merge pull request #23455 from eileenmcnaughton/import_validate_mum

Import validate related contacts in the same way as the main contact

CRM/Contact/Import/Parser/Contact.php

@@ -29,14 +29,8 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser {
   use CRM_Contact_Import_MetadataTrait;
 
   protected $_mapperKeys = [];
-  protected $_mapperRelated;
-  protected $_mapperRelatedContactDetails;
   protected $_relationships;
 
-  protected $_emailIndex;
-
-  protected $_phoneIndex;
-
   /**
    * Is update only permitted on an id match.
    *
@@ -116,19 +110,10 @@ class CRM_Contact_Import_Parser_Contact extends CRM_Import_Parser {
    * Class constructor.
    *
    * @param array $mapperKeys
-   * @param array $mapperLocType
-   * @param array $mapperPhoneType
-   * @param array $mapperImProvider
-   * @param array $mapperRelated
-   * @param array $mapperRelatedContactType
-   * @param array $mapperRelatedContactDetails
    */
-  public function __construct(
-    $mapperKeys = [], $mapperLocType = [], $mapperPhoneType = [], $mapperImProvider = [], $mapperRelated = [], $mapperRelatedContactType = [], $mapperRelatedContactDetails = []) {
+  public function __construct($mapperKeys = []) {
     parent::__construct();
     $this->_mapperKeys = $mapperKeys;
-    $this->_mapperRelated = &$mapperRelated;
-    $this->_mapperRelatedContactDetails = &$mapperRelatedContactDetails;
   }
 
   /**
@@ -143,35 +128,38 @@ public function init() {
 
     $this->setActiveFields($this->_mapperKeys);
 
-    $this->_phoneIndex = -1;
-    $this->_emailIndex = -1;
     $this->_externalIdentifierIndex = -1;
 
     $index = 0;
     foreach ($this->_mapperKeys as $key) {
-      if (substr($key, 0, 5) == 'email' && substr($key, 0, 14) != 'email_greeting') {
-        $this->_emailIndex = $index;
-      }
-      if (substr($key, 0, 5) == 'phone') {
-        $this->_phoneIndex = $index;
-      }
       if ($key == 'external_identifier') {
         $this->_externalIdentifierIndex = $index;
       }
       $index++;
     }
 
     $this->_updateWithId = FALSE;
-    if (in_array('id', $this->_mapperKeys) || ($this->_externalIdentifierIndex >= 0 && in_array($this->_onDuplicate, [
-      CRM_Import_Parser::DUPLICATE_UPDATE,
-      CRM_Import_Parser::DUPLICATE_FILL,
-    ]))) {
+    if (in_array('id', $this->_mapperKeys) || ($this->_externalIdentifierIndex >= 0 && $this->isUpdateExistingContacts())) {
       $this->_updateWithId = TRUE;
     }
 
     $this->_parseStreetAddress = CRM_Utils_Array::value('street_address_parsing', CRM_Core_BAO_Setting::valueOptions(CRM_Core_BAO_Setting::SYSTEM_PREFERENCES_NAME, 'address_options'), FALSE);
   }
 
+  /**
+   * Is this a case where the user has opted to update existing contacts.
+   *
+   * @return bool
+   *
+   * @throws \API_Exception
+   */
+  private function isUpdateExistingContacts(): bool {
+    return in_array((int) $this->getSubmittedValue('onDuplicate'), [
+      CRM_Import_Parser::DUPLICATE_UPDATE,
+      CRM_Import_Parser::DUPLICATE_FILL,
+    ], TRUE);
+  }
+
   /**
    * Gets the fields available for importing in a key-name, title format.
    *
@@ -645,15 +633,6 @@ public function import($onDuplicate, &$values) {
           //format common data, CRM-4062
           $this->formatCommonData($field, $formatting, $contactFields);
 
-          //do we have enough fields to create related contact.
-          $allowToCreate = $this->checkRelatedContactFields($key, $formatting);
-
-          if (!$allowToCreate) {
-            $errorMessage = ts('Related contact required fields are missing.');
-            array_unshift($values, $errorMessage);
-            return CRM_Import_Parser::NO_MATCH;
-          }
-
           //fixed for CRM-4148
           if (!empty($params[$key]['id'])) {
             $contact = [
@@ -1930,50 +1909,6 @@ public function processMessage(&$values, $statusFieldName, $returnCode) {
     return $returnCode;
   }
 
-  /**
-   * @param $relKey
-   * @param array $params
-   *
-   * @return bool
-   */
-  public function checkRelatedContactFields($relKey, $params) {
-    //avoid blank contact creation.
-    $allowToCreate = FALSE;
-
-    //build the mapper field array.
-    static $relatedContactFields = [];
-    if (!isset($relatedContactFields[$relKey])) {
-      foreach ($this->_mapperRelated as $key => $name) {
-        if (!$name) {
-          continue;
-        }
-
-        if (!empty($relatedContactFields[$name]) && !is_array($relatedContactFields[$name])) {
-          $relatedContactFields[$name] = [];
-        }
-        $fldName = $this->_mapperRelatedContactDetails[$key] ?? NULL;
-        if ($fldName == 'url') {
-          $fldName = 'website';
-        }
-        if ($fldName) {
-          $relatedContactFields[$name][] = $fldName;
-        }
-      }
-    }
-
-    //validate for passed data.
-    if (is_array($relatedContactFields[$relKey])) {
-      foreach ($relatedContactFields[$relKey] as $fld) {
-        if (!empty($params[$fld])) {
-          $allowToCreate = TRUE;
-          break;
-        }
-      }
-    }
-
-    return $allowToCreate;
-  }
-
   /**
    * get subtypes given the contact type
    *
@@ -3062,66 +2997,23 @@ public function getMappedRow(array $values): array {
    * @param array $values
    *
    * @throws \API_Exception
+   * @throws \CRM_Core_Exception
    */
   public function validateValues(array $values): void {
-    $errorMessage = NULL;
-    $errorRequired = FALSE;
     $params = $this->getMappedRow($values);
-    $missingNames = [];
-    switch ($params['contact_type']) {
-      case 'Individual':
-        if (empty($params['first_name'])) {
-          $missingNames[] = ts('First Name');
-        }
-        if (empty($params['last_name'])) {
-          $missingNames[] = ts('Last Name');
-        }
-        break;
-
-      case 'Household':
-        if (empty($params['household_name'])) {
-          $missingNames[] = ts('Missing required fields:') . ' ' . ts('Household Name');
-        }
-        break;
-
-      case 'Organization':
-        if (empty($params['organization_name'])) {
-          $missingNames[] = ts('Missing required fields:') . ' ' . ts('Organization Name');
-        }
-        break;
-    }
-    if (!empty($missingNames)) {
-      $errorMessage = ts('Missing required fields:') . ' ' . implode(' ' . ts('and') . ' ', $missingNames);
-      $errorRequired = TRUE;
-    }
-
-    if ($this->_emailIndex >= 0) {
-      /* If we don't have the required fields, bail */
-
-      if ($this->_contactType === 'Individual' && !$this->_updateWithId) {
-        if ($errorRequired && empty($values[$this->_emailIndex])) {
-          if ($errorMessage) {
-            $errorMessage .= ' ' . ts('OR') . ' ' . ts('Email Address');
-          }
-          else {
-            $errorMessage = ts('Missing required field:') . ' ' . ts('Email Address');
-          }
-          throw new CRM_Core_Exception($errorMessage);
-        }
-      }
-    }
-    elseif ($errorRequired && !$this->_updateWithId) {
-      if ($errorMessage) {
-        $errorMessage .= ' ' . ts('OR') . ' ' . ts('Email Address');
-      }
-      else {
-        $errorMessage = ts('Missing required field:') . ' ' . ts('Email Address');
+    $this->validateRequiredContactFields($params['contact_type'], $params, $this->isUpdateExistingContacts());
+    foreach ($params as $key => $value) {
+      // If the key is a relationship key - eg. 5_a_b or 10_b_a
+      // then the value is an array that describes an existing contact.
+      // We need to check the fields are present to identify or create this
+      // contact.
+      if (preg_match('/^\d+_[a|b]_[a|b]$/', $key)) {
+        $this->validateRequiredContactFields($value['contact_type'], $value, TRUE, '(' . $this->getRelatedContactLabel(substr($key, 0, -4), substr($key, -3)) . ')');
       }
-      throw new CRM_Core_Exception($errorMessage);
     }
 
     //check for duplicate external Identifier
-    $externalID = $values[$this->_externalIdentifierIndex] ?? NULL;
+    $externalID = $params['external_identifier'] ?? NULL;
     if ($externalID) {
       /* If it's a dupe,external Identifier  */
 
@@ -3197,12 +3089,39 @@ protected function getRelatedContactType($relationshipTypeID, $relationshipDirec
     if (!$relationshipTypeID) {
       return NULL;
     }
-    $cacheKey = $relationshipTypeID . $relationshipDirection;
+    $relationshipField = 'contact_type_' . substr($relationshipDirection, -1);
+    return $this->getRelationshipType($relationshipTypeID, $relationshipDirection)[$relationshipField];
+  }
+
+  /**
+   * Get the related contact type.
+   *
+   * @param int|null $relationshipTypeID
+   * @param int|string $relationshipDirection
+   *
+   * @return null|string
+   *
+   * @throws \API_Exception
+   */
+  protected function getRelatedContactLabel($relationshipTypeID, $relationshipDirection): ?string {
+    $relationshipField = 'label_' . $relationshipDirection;
+    return $this->getRelationshipType($relationshipTypeID, $relationshipDirection)[$relationshipField];
+  }
+
+  /**
+   * Get the relationship type.
+   *
+   * @param int $relationshipTypeID
+   *
+   * @return string[]
+   * @throws \API_Exception
+   */
+  protected function getRelationshipType(int $relationshipTypeID): array {
+    $cacheKey = 'relationship_type' . $relationshipTypeID;
     if (!isset(Civi::$statics[__CLASS__][$cacheKey])) {
-      $relationshipField = 'contact_type_' . substr($relationshipDirection, -1);
       Civi::$statics[__CLASS__][$cacheKey] = RelationshipType::get(FALSE)
         ->addWhere('id', '=', $relationshipTypeID)
-        ->addSelect($relationshipField)->execute()->first()[$relationshipField];
+        ->addSelect('*')->execute()->first();
     }
     return Civi::$statics[__CLASS__][$cacheKey];
   }

CRM/Import/Parser.php

@@ -498,6 +498,52 @@ public function setMaxLinesToProcess($max) {
     $this->_maxLinesToProcess = $max;
   }
 
+  /**
+   * Validate that we have the required fields to create the contact or find it to update.
+   *
+   * Note that the users duplicate selection affects this as follows
+   * - if they did not select an update variant then the id field is not
+   *   permitted in the mapping - so we can assume the presence of id means
+   *   we should use it
+   * - the external_identifier field is valid in place of the other fields
+   *   when they have chosen update or fill - in this case we are only looking
+   *   to update an existing contact.
+   *
+   * @param string $contactType
+   * @param array $params
+   * @param bool $isPermitExistingMatchFields
+   *   True if the it is enough to have fields which will enable us to find
+   *   an existing contact (eg. external_identifier).
+   * @param string $prefixString
+   *   String to include in the exception (e.g '(Child of)' if we are validating
+   *   a related contact.
+   *
+   * @return void
+   * @throws \CRM_Core_Exception
+   */
+  protected function validateRequiredContactFields(string $contactType, array $params, bool $isPermitExistingMatchFields = TRUE, $prefixString = ''): void {
+    if (!empty($params['id'])) {
+      return;
+    }
+    $requiredFields = [
+      'Individual' => [
+        'first_name_last_name' => ['first_name' => ts('First Name'), 'last_name' => ts('Last Name')],
+        'email' => ts('Email Address'),
+      ],
+      'Organization' => ['organization_name' => ts('Organization Name')],
+      'Household' => ['household_name' => ts('Household Name')],
+    ][$contactType];
+    if ($isPermitExistingMatchFields) {
+      $requiredFields['external_identifier'] = ts('External Identifier');
+      // Historically just an email has been accepted as it is 'usually good enough'
+      // for a dedupe rule look up - but really this is a stand in for
+      // whatever is needed to find an existing matching contact using the
+      // specified dedupe rule (or the default Unsupervised if not specified).
+      $requiredFields['email'] = ts('Email Address');
+    }
+    $this->validateRequiredFields($requiredFields, $params, $prefixString);
+  }
+
   /**
    * Determines the file extension based on error code.
    *
@@ -1073,4 +1119,56 @@ protected function getIdsOfMatchingContacts(array $formatted):array {
     }
   }
 
+  /**
+   * Validate that the field requirements are met in the params.
+   *
+   * @param array $requiredFields
+   * @param array $params
+   *   An array of required fields (fieldName => label)
+   *   - note this follows the and / or array nesting we see in permission checks
+   *   eg.
+   *   [
+   *     'email' => ts('Email'),
+   *     ['first_name' => ts('First Name'), 'last_name' => ts('Last Name')]
+   *   ]
+   *   Means 'email' OR 'first_name AND 'last_name'.
+   * @param string $prefixString
+   *
+   * @throws \CRM_Core_Exception Exception thrown if field requirements are not met.
+   */
+  protected function validateRequiredFields(array $requiredFields, array $params, $prefixString): void {
+    $missingFields = [];
+    foreach ($requiredFields as $key => $required) {
+      if (!is_array($required)) {
+        $importParameter = $params[$key] ?? [];
+        if (!is_array($importParameter)) {
+          if (!empty($importParameter)) {
+            return;
+          }
+        }
+        else {
+          foreach ($importParameter as $locationValues) {
+            if (!empty($locationValues[$key])) {
+              return;
+            }
+          }
+        }
+
+        $missingFields[$key] = $required;
+      }
+      else {
+        foreach ($required as $field => $label) {
+          if (empty($params[$field])) {
+            $missing[$field] = $label;
+          }
+        }
+        if (empty($missing)) {
+          return;
+        }
+        $missingFields[$key] = implode(' ' . ts('and') . ' ', $missing);
+      }
+    }
+    throw new CRM_Core_Exception(($prefixString ? ($prefixString . ' ') : '') . ts('Missing required fields:') . ' ' . implode(' ' . ts('OR') . ' ', $missingFields));
+  }
+
 }

tests/phpunit/CRM/Contact/Import/Form/data/individual_invalid_external_identifier_only.csv

@@ -0,0 +1,2 @@
+External Identifier,Gender
+1234,Female

tests/phpunit/CRM/Contact/Import/Form/data/individual_invalid_with_related_phone.csv

@@ -0,0 +1,2 @@
+Player First Name,Player Last name,Mother – phone
+Susie,Jones,911

tests/phpunit/CRM/Contact/Import/Form/data/organization_email_no_organization_name.csv

@@ -0,0 +1,2 @@
+Email,Phone
+my-org@example.com,0123 111 111