Membership form flow tidy-up

CRM/Activity/Import/Form/DataSource.php

@@ -60,4 +60,13 @@ protected function getParser(): CRM_Activity_Import_Parser_Activity {
     return $this->parser;
   }
 
+  /**
+   * Get the name of the type to be stored in civicrm_user_job.type_id.
+   *
+   * @return string
+   */
+  public function getUserJobType(): string {
+    return 'activity_import';
+  }
+
 }

CRM/Contact/Import/Form/DataSource.php

@@ -220,4 +220,13 @@ public function getTitle(): string {
     return ts('Choose Data Source');
   }
 
+  /**
+   * Get the name of the type to be stored in civicrm_user_job.type_id.
+   *
+   * @return string
+   */
+  public function getUserJobType(): string {
+    return 'contact_import';
+  }
+
 }

CRM/Contact/Import/Form/Summary.php

@@ -47,16 +47,7 @@ public function preProcess() {
 
     $this->assign('groupAdditions', $this->getUserJob()['metadata']['summary_info']['groups']);
     $this->assign('tagAdditions', $this->getUserJob()['metadata']['summary_info']['tags']);
-    $this->assign('totalRowCount', $this->getRowCount());
-    $this->assign('validRowCount', $this->getRowCount(CRM_Import_Parser::VALID) + $this->getRowCount(CRM_Import_Parser::UNPARSED_ADDRESS_WARNING));
-    $this->assign('invalidRowCount', $this->getRowCount(CRM_Import_Parser::ERROR));
-    $this->assign('duplicateRowCount', $this->getRowCount(CRM_Import_Parser::DUPLICATE));
-    $this->assign('unMatchCount', $this->getRowCount(CRM_Import_Parser::NO_MATCH));
-    $this->assign('unparsedAddressCount', $this->getRowCount(CRM_Import_Parser::UNPARSED_ADDRESS_WARNING));
-    $this->assign('downloadDuplicateRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::DUPLICATE));
-    $this->assign('downloadErrorRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::ERROR));
-    $this->assign('downloadMismatchRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::NO_MATCH));
-    $this->assign('downloadAddressRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::UNPARSED_ADDRESS_WARNING));
+    $this->assignOutputURLs();
     $session = CRM_Core_Session::singleton();
     $session->pushUserContext(CRM_Utils_System::url('civicrm/import/contact', 'reset=1'));
   }

CRM/Contribute/Import/Form/DataSource.php

@@ -71,4 +71,13 @@ protected function getParser(): CRM_Contribute_Import_Parser_Contribution {
     return $this->parser;
   }
 
+  /**
+   * Get the name of the type to be stored in civicrm_user_job.type_id.
+   *
+   * @return string
+   */
+  public function getUserJobType(): string {
+    return 'contribution_import';
+  }
+
 }

CRM/Core/BAO/UserJob.php

@@ -82,6 +82,37 @@ public static function getTypes(): array {
         'id' => 1,
         'name' => 'contact_import',
         'label' => ts('Contact Import'),
+        'class' => 'CRM_Contact_Import_Parser_Contact',
+      ],
+      [
+        'id' => 2,
+        'name' => 'contribution_import',
+        'label' => ts('Contribution Import'),
+        'class' => 'CRM_Contribute_Import_Parser_Contribution',
+      ],
+      [
+        'id' => 3,
+        'name' => 'membership_import',
+        'label' => ts('Membership Import'),
+        'class' => 'CRM_Member_Import_Parser_Membership',
+      ],
+      [
+        'id' => 4,
+        'name' => 'activity_import',
+        'label' => ts('Activity Import'),
+        'class' => 'CRM_Activity_Import_Parser_Activity',
+      ],
+      [
+        'id' => 5,
+        'name' => 'participant_import',
+        'label' => ts('Participant Import'),
+        'class' => 'CRM_Event_Import_Parser_Participant',
+      ],
+      [
+        'id' => 6,
+        'name' => 'custom_field_import',
+        'label' => ts('Multiple Value Custom Field Import'),
+        'class' => 'CRM_Custom_Import_Parser_Api',
       ],
     ];
   }

CRM/Custom/Import/Form/DataSource.php

@@ -108,4 +108,13 @@ protected function getParser(): CRM_Custom_Import_Parser_Api {
     return $this->parser;
   }
 
+  /**
+   * Get the name of the type to be stored in civicrm_user_job.type_id.
+   *
+   * @return string
+   */
+  public function getUserJobType(): string {
+    return 'contact_import';
+  }
+
 }

CRM/Import/Form/MapField.php

@@ -76,6 +76,22 @@ public function preProcess() {
     parent::preProcess();
   }
 
+  /**
+   * Process the mapped fields and map it into the uploaded file
+   * preview the file and extract some summary statistics
+   *
+   * @return void
+   * @noinspection PhpDocSignatureInspection
+   * @noinspection PhpUnhandledExceptionInspection
+   */
+  public function postProcess() {
+    $this->updateUserJobMetadata('submitted_values', $this->getSubmittedValues());
+    $this->saveMapping($this->getMappingTypeName());
+    $parser = $this->getParser();
+    $parser->init();
+    $parser->validate();
+  }
+
   /**
    * Attempt to match header labels with our mapper fields.
    *
@@ -230,6 +246,9 @@ protected function getMappedField(array $fieldMapping, int $mappingID, int $colu
   protected function saveMappingField(int $mappingID, int $columnNumber, bool $isUpdate = FALSE): void {
     $fieldMapping = (array) $this->getSubmittedValue('mapper')[$columnNumber];
     $mappedField = $this->getMappedField($fieldMapping, $mappingID, $columnNumber);
+    if (empty($mappedField['name'])) {
+      $mappedField['name'] = 'do_not_import';
+    }
     if ($isUpdate) {
       Civi\Api4\MappingField::update(FALSE)
         ->setValues($mappedField)

CRM/Import/Form/Preview.php

@@ -117,4 +117,14 @@ protected function assignPreviewVariables(): void {
     $this->assign('rowDisplayCount', $this->getSubmittedValue('skipColumnHeader') ? 3 : 2);
   }
 
+  /**
+   * Process the mapped fields and map it into the uploaded file
+   * preview the file and extract some summary statistics
+   *
+   * @return void
+   */
+  public function postProcess() {
+    CRM_Import_Parser::runImport(NULL, $this->getUserJobID(), 0);
+  }
+
 }

CRM/Import/Form/Summary.php

@@ -23,6 +23,15 @@
  */
 abstract class CRM_Import_Form_Summary extends CRM_Import_Forms {
 
+  /**
+   * Set variables up before form is built.
+   *
+   * @return void
+   */
+  public function preProcess() {
+    $this->assignOutputURLs();
+  }
+
   /**
    * Build the form object.
    */
@@ -45,4 +54,17 @@ public function getTitle() {
     return ts('Summary');
   }
 
+  protected function assignOutputURLs(): void {
+    $this->assign('totalRowCount', $this->getRowCount());
+    $this->assign('validRowCount', $this->getRowCount(CRM_Import_Parser::VALID) + $this->getRowCount(CRM_Import_Parser::UNPARSED_ADDRESS_WARNING));
+    $this->assign('invalidRowCount', $this->getRowCount(CRM_Import_Parser::ERROR));
+    $this->assign('duplicateRowCount', $this->getRowCount(CRM_Import_Parser::DUPLICATE));
+    $this->assign('unMatchCount', $this->getRowCount(CRM_Import_Parser::NO_MATCH));
+    $this->assign('unparsedAddressCount', $this->getRowCount(CRM_Import_Parser::UNPARSED_ADDRESS_WARNING));
+    $this->assign('downloadDuplicateRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::DUPLICATE));
+    $this->assign('downloadErrorRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::ERROR));
+    $this->assign('downloadMismatchRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::NO_MATCH));
+    $this->assign('downloadAddressRecordsUrl', $this->getDownloadURL(CRM_Import_Parser::UNPARSED_ADDRESS_WARNING));
+  }
+
 }

CRM/Import/Forms.php

@@ -376,7 +376,7 @@ protected function createUserJob(): int {
     $id = UserJob::create(FALSE)
       ->setValues([
         'created_id' => CRM_Core_Session::getLoggedInContactID(),
-        'type_id:name' => 'contact_import',
+        'type_id:name' => $this->getUserJobType(),
         'status_id:name' => 'draft',
         // This suggests the data could be cleaned up after this.
         'expires_date' => '+ 1 week',

CRM/Import/Parser.php

@@ -625,8 +625,8 @@ protected function validateRequiredContactFields(string $contactType, array $par
 
   protected function doPostImportActions() {
     $userJob = $this->getUserJob();
-    $summaryInfo = $userJob['metadata']['summary_info'];
-    $actions = $userJob['metadata']['post_actions'];
+    $summaryInfo = $userJob['metadata']['summary_info'] ?? [];
+    $actions = $userJob['metadata']['post_actions'] ?? [];
     if (!empty($actions['group'])) {
       $groupAdditions = $this->addImportedContactsToNewGroup($this->createdContacts, $actions['group']);
       foreach ($actions['group'] as $groupID) {
@@ -1543,7 +1543,7 @@ public function validate(): void {
   protected function getInvalidValues($value, string $key = '', string $prefixString = ''): array {
     $errors = [];
     if ($value === 'invalid_import_value') {
-      $errors[] = $prefixString . $this->getFieldMetadata($key)['title'];
+      $errors[] = $prefixString . $this->getFieldMetadata($key)['html']['label'] ?? $this->getFieldMetadata($key)['title'];
     }
     elseif (is_array($value)) {
       foreach ($value as $innerKey => $innerValue) {
@@ -1634,6 +1634,40 @@ public function getMappingFieldFromMapperInput(array $fieldMapping, int $mapping
     ];
   }
 
+  /**
+   * @param array $mappedField
+   *   Field detail as would be saved in field_mapping table
+   *   or as returned from getMappingFieldFromMapperInput
+   *
+   * @return string
+   * @throws \API_Exception
+   */
+  public function getMappedFieldLabel(array $mappedField): string {
+    $this->setFieldMetadata();
+    return $this->getFieldMetadata($mappedField['name'])['title'];
+  }
+
+  /**
+   * Get the row from the csv mapped to our parameters.
+   *
+   * @param array $values
+   *
+   * @return array
+   * @throws \API_Exception
+   */
+  public function getMappedRow(array $values): array {
+    $params = [];
+    foreach ($this->getFieldMappings() as $i => $mappedField) {
+      if ($mappedField['name'] === 'do_not_import') {
+        continue;
+      }
+      if ($mappedField['name']) {
+        $params[$this->getFieldMetadata($mappedField['name'])['name']] = $this->getTransformedFieldValue($mappedField['name'], $values[$i]);
+      }
+    }
+    return $params;
+  }
+
   /**
    * Get the field mappings for the import.
    *
@@ -1655,6 +1689,52 @@ protected function getFieldMappings(): array {
     return $mappedFields;
   }
 
+  /**
+   * Run import.
+   *
+   * @param \CRM_Queue_TaskContext $taskContext
+   *
+   * @param int $userJobID
+   * @param int $limit
+   *
+   * @return bool
+   * @throws \API_Exception
+   * @throws \CRM_Core_Exception
+   */
+  public static function runImport($taskContext, $userJobID, $limit) {
+    $userJob = UserJob::get()->addWhere('id', '=', $userJobID)->addSelect('type_id')->execute()->first();
+    $parserClass = NULL;
+    foreach (CRM_Core_BAO_UserJob::getTypes() as $userJobType) {
+      if ($userJob['type_id'] === $userJobType['id']) {
+        $parserClass = $userJobType['class'];
+      }
+    }
+    $parser = new $parserClass();
+    $parser->setUserJobID($userJobID);
+    // Not sure if we still need to init....
+    $parser->init();
+    $dataSource = $parser->getDataSourceObject();
+    $dataSource->setStatuses(['new']);
+    $dataSource->setLimit($limit);
+
+    while ($row = $dataSource->getRow()) {
+      $values = array_values($row);
+
+      try {
+        $parser->import($parser->getSubmittedValue('onDuplicate'), $values);
+      }
+      catch (CiviCRM_API3_Exception $e) {
+        // When we catch errors here we are not adding to the errors array - mostly
+        // because that will become obsolete once https://github.com/civicrm/civicrm-core/pull/23292
+        // is merged and this will replace it as the main way to handle errors (ie. update the table
+        // and move on).
+        $parser->setImportStatus((int) $values[count($values) - 1], 'ERROR', $e->getMessage());
+      }
+    }
+    $parser->doPostImportActions();
+    return TRUE;
+  }
+
   /**
    * Check if an error in custom data.
    *

CRM/Member/Import/Form/DataSource.php

@@ -71,4 +71,13 @@ protected function getParser(): CRM_Member_Import_Parser_Membership {
     return $this->parser;
   }
 
+  /**
+   * Get the name of the type to be stored in civicrm_user_job.type_id.
+   *
+   * @return string
+   */
+  public function getUserJobType(): string {
+    return 'membership_import';
+  }
+
 }