Merge pull request #24510 from eileenmcnaughton/import_default

Add unit test & handling  for processing from UserJob configuration rather than form submitted values

CRM/Contribute/Import/Parser/Contribution.php

@@ -127,19 +127,36 @@ public static function getUserJobInfo(): array {
   /**
    * Get the field mappings for the import.
    *
-   * This is the same format as saved in civicrm_mapping_field except
-   * that location_type_id = 'Primary' rather than empty where relevant.
-   * Also 'im_provider_id' is mapped to the 'real' field name 'provider_id'
-   *
    * @return array
+   *   Array of arrays with each array representing a row in the datasource.
+   *   The arrays hold the following keys
+   *   - name - field the row maps to
+   *   - entity_data - data about the relevant entity ie ['soft_credit' => ['soft_credit_type_id => 9],
+   *   In addition the following are returned but will be phased out.
+   *   - contact_type - entity_data but json_encoded. Saved to civicrm_mapping_field in contact_type column
+   *   - column_number = this is used for saving to civicrm_field_mapping but
+   *     may be only legacy now?
+   *   - soft_credit_type_id
+   *
+   * @throws \CRM_Core_Exception
    */
   protected function getFieldMappings(): array {
-    $mappedFields = [];
-    foreach ($this->getSubmittedValue('mapper') as $i => $mapperRow) {
-      $mappedField = $this->getMappingFieldFromMapperInput($mapperRow, 0, $i);
-      // Just for clarity since 0 is a pseudo-value
-      unset($mappedField['mapping_id']);
-      $mappedFields[] = $mappedField;
+    $mappedFields = $this->getUserJob()['metadata']['import mappings'] ?? [];
+    if (empty($mappedFields)) {
+      foreach ($this->getSubmittedValue('mapper') as $i => $mapperRow) {
+        $mappedField = $this->getMappingFieldFromMapperInput($mapperRow, 0, $i);
+        // Just for clarity since 0 is a pseudo-value
+        unset($mappedField['mapping_id']);
+        $mappedFields[] = $mappedField;
+      }
+    }
+    foreach ($mappedFields as $index => $mappedField) {
+      $mappedFields[$index]['column_number'] = 0;
+      // This is the same data as entity_data - it is stored to the database in the contact_type field
+      // slit your eyes & squint while blinking and you can almost read that as entity_type and not
+      // hate it. Otherwise go & whinge on https://lab.civicrm.org/dev/core/-/issues/1172
+      $mappedFields[$index]['contact_type'] = !empty($mappedField['entity_type']) ? json_encode($mappedField['entity_type']) : NULL;
+      $mappedFields[$index]['soft_credit_type_id'] = !empty($mappedField['entity_type']) ? $mappedField['entity_type']['soft_credit']['soft_credit_type_id'] : NULL;
     }
     return $mappedFields;
   }
@@ -195,19 +212,23 @@ public function getMappedRow(array $values): array {
         continue;
       }
       $fieldSpec = $this->getFieldMetadata($mappedField['name']);
+      $fieldValue = $values[$i];
+      if ($fieldValue === '' && isset($mappedField['default_value'])) {
+        $fieldValue = $mappedField['default_value'];
+      }
       $entity = $fieldSpec['entity_instance'] ?? ($fieldSpec['entity'] ?? 'Contribution');
       // If we move this to the parent we can check if the entity config 'supports_multiple'
       if ($entity === 'SoftCreditContact') {
         $entityKey = json_encode($mappedField['entity_data']);
         $entityInstance = $params[$entity][$entityKey] ?? $mappedField['entity_data']['soft_credit'];
-        $entityInstance['Contact'] = array_merge($entityInstance['Contact'] ?? [], [$this->getFieldMetadata($mappedField['name'])['name'] => $this->getTransformedFieldValue($mappedField['name'], $values[$i])]);
+        $entityInstance['Contact'] = array_merge($entityInstance['Contact'] ?? [], [$this->getFieldMetadata($mappedField['name'])['name'] => $this->getTransformedFieldValue($mappedField['name'], $fieldValue)]);
         $params[$entity][$entityKey] = $entityInstance;
       }
       else {
         if ($entity === 'Contact' && !isset($params[$entity])) {
           $params[$entity] = $this->getContactType() ? ['contact_type' => $this->getContactType()] : [];
         }
-        $params[$entity][$this->getFieldMetadata($mappedField['name'])['name']] = $this->getTransformedFieldValue($mappedField['name'], $values[$i]);
+        $params[$entity][$this->getFieldMetadata($mappedField['name'])['name']] = $this->getTransformedFieldValue($mappedField['name'], $fieldValue);
       }
     }
     return $params;
@@ -631,12 +652,7 @@ public function getMappingFieldFromMapperInput(array $fieldMapping, int $mapping
       'name' => str_replace('__', '.', $fieldMapping[0]),
       'mapping_id' => $mappingID,
       'column_number' => $columnNumber,
-      'soft_credit_type_id' => $fieldMapping[1] ?? NULL,
       'entity_data' => !empty($fieldMapping[1]) ? ['soft_credit' => ['soft_credit_type_id' => $fieldMapping[1]]] : NULL,
-      // This is the same data as entity_data - it is stored to the database in the contact_type field
-      // slit your eyes & squint while blinking and you can almost read that as entity_type and not
-      // hate it. Otherwise go & whinge on https://lab.civicrm.org/dev/core/-/issues/1172
-      'contact_type' => !empty($fieldMapping[1]) ? json_encode(['soft_credit' => ['soft_credit_type_id' => $fieldMapping[1]]]) : NULL,
     ];
   }
 
@@ -728,8 +744,8 @@ public function getMappedFieldLabel(array $mappedField): string {
     }
     $title = [];
     $title[] = $this->getFieldMetadata($mappedField['name'])['title'];
-    if ($mappedField['soft_credit_type_id']) {
-      $title[] = CRM_Core_PseudoConstant::getLabel('CRM_Contribute_BAO_ContributionSoft', 'soft_credit_type_id', $mappedField['soft_credit_type_id']);
+    if (isset($mappedField['soft_credit'])) {
+      $title[] = CRM_Core_PseudoConstant::getLabel('CRM_Contribute_BAO_ContributionSoft', 'soft_credit_type_id', $mappedField['soft_credit']['soft_credit_type_id']);
     }
 
     return implode(' - ', $title);

tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php

@@ -188,6 +188,55 @@ public function testContributionStatusLabel(): void {
 
   }
 
+  /**
+   * Test the an import can be done based on saved configuration in the UserJob.
+   *
+   * This also demonstrates some advanced import handling that the quickForm
+   * layer does not support but if you can get the config INTO the user_job
+   * table it runs... (ie via the angular form).
+   *
+   * These features are
+   *  - default_value for each field.
+   *
+   * @throws \API_Exception
+   */
+  public function testImportFromUserJobConfiguration(): void {
+    $importMappings = [
+      ['name' => 'organization_name'],
+      ['name' => 'total_amount'],
+      // Note that default_value is supported via the parser and the angular form
+      // but there is no way to enter it on the quick form.
+      ['name' => 'financial_type_id', 'default_value' => 'Donation'],
+      ['name' => 'contribution_source'],
+      ['name' => 'receive_date'],
+      ['name' => 'soft_credit.contact.email_primary.email', 'entity_data' => ['soft_credit' => ['soft_credit_type_id' => 5]]],
+      ['name' => 'soft_credit.contact.first_name', 'entity_data' => ['soft_credit' => ['soft_credit_type_id' => 5]]],
+      ['name' => 'soft_credit.contact.last_name', 'entity_data' => ['soft_credit' => ['soft_credit_type_id' => 5]]],
+    ];
+    $submittedValues = [
+      'skipColumnHeader' => TRUE,
+      'fieldSeparator' => ',',
+      'contactType' => 'Organization',
+      'mapper' => $this->getMapperFromFieldMappings($importMappings),
+      'dataSource' => 'CRM_Import_DataSource_CSV',
+      'dateFormats' => CRM_Core_Form_Date::DATE_yyyy_mm_dd,
+      'onDuplicate' => CRM_Import_Parser::DUPLICATE_SKIP,
+    ];
+    $this->submitDataSourceForm('soft_credit_extended.csv', $submittedValues);
+    $metadata = UserJob::get()->addWhere('id', '=', $this->userJobID)->addSelect('metadata')->execute()->first()['metadata'];
+    $metadata['import mappings'] = $importMappings;
+    UserJob::update()->addWhere('id', '=', $this->userJobID)
+      ->setValues(['metadata' => $metadata])->execute();
+    $form = $this->getMapFieldForm($submittedValues);
+    $form->setUserJobID($this->userJobID);
+    $form->buildForm();
+    $this->assertTrue($form->validate());
+    $form->postProcess();
+    $row = $this->getDataSource()->getRow();
+    // a valid status here means it has been able to incorporate the default_value.
+    $this->assertEquals('VALID', $row['_status']);
+  }
+
   /**
    * Test dates are parsed.
    */

tests/phpunit/CRM/Contribute/Import/Parser/data/soft_credit_extended.csv

@@ -0,0 +1,3 @@
+Organization Name,Amount,Financial Type,Source,Date,Soft credi contact email,Soft credit first name,Soft Credit last name
+Big Firm,800,,Import,2022-09-08,jenny@example.org,Jenny,Hawthorn
+Small Firm,70,Check,Import,2022-09-08,sarah@example.org,Sarah,Windsor