Add import & validate actions

civicrm · Sep 26, 2022 · 9f168b3 · 9f168b3
1 parent 4ecdb51
commit 9f168b3
Show file tree

Hide file tree

Showing 7 changed files with 181 additions and 16 deletions.
diff --git a/CRM/Import/Parser.php b/CRM/Import/Parser.php
@@ -632,7 +632,12 @@ protected function getRequiredFieldsContactCreate(): array {
     ];
   }
 
-  protected function doPostImportActions() {
+  /**
+   * Core function - do not call from outside core.
+   *
+   * @internal
+   */
+  public function doPostImportActions() {
     $userJob = $this->getUserJob();
     $summaryInfo = $userJob['metadata']['summary_info'] ?? [];
     $actions = $userJob['metadata']['post_actions'] ?? [];

diff --git a/ext/civiimport/Civi/Api4/Import.php b/ext/civiimport/Civi/Api4/Import.php
@@ -17,6 +17,8 @@
 use Civi\Api4\Import\Create;
 use Civi\Api4\Import\Save;
 use Civi\Api4\Import\Update;
+use Civi\Api4\Import\Import as ImportAction;
+use Civi\Api4\Import\Validate;
 
 /**
  * Import entity.
@@ -63,7 +65,8 @@ public static function save(int $userJobID, bool $checkPermissions = TRUE): Save
   /**
    * @param int $userJobID
    * @param bool $checkPermissions
-   * @return \Civi\Api4\Generic\DAOCreateAction
+   *
+   * @return \Civi\Api4\Import\Create
    * @throws \API_Exception
    */
   public static function create(int $userJobID, bool $checkPermissions = TRUE): Create {
@@ -101,6 +104,30 @@ public static function checkAccess(int $userJobID): CheckAccessAction {
     return new CheckAccessAction('Import_' . $userJobID, __FUNCTION__);
   }
 
+  /**
+   * @param int $userJobID
+   * @param bool $checkPermissions
+   *
+   * @return \Civi\Api4\Import\Import
+   *
+   * @throws \API_Exception
+   */
+  public static function import(int $userJobID, bool $checkPermissions = TRUE) {
+    return (new ImportAction('Import_' . $userJobID, __FUNCTION__))
+      ->setCheckPermissions($checkPermissions);
+  }
+
+  /**
+   * @param int $userJobID
+   * @param bool $checkPermissions
+   *
+   * @return \Civi\Api4\Import\Validate
+   * @throws \API_Exception
+   */
+  public static function validate(int $userJobID, bool $checkPermissions = TRUE): Validate {
+    return (new Validate('Import_' . $userJobID, __FUNCTION__))->setCheckPermissions($checkPermissions);
+  }
+
   /**
    * We need to implement these elsewhere as we permit based on 'created_id'.
    *

diff --git a/ext/civiimport/Civi/Api4/Import/Import.php b/ext/civiimport/Civi/Api4/Import/Import.php
@@ -1 +1,42 @@
 <?php
+
+/*
+ +--------------------------------------------------------------------+
+ | Copyright CiviCRM LLC. All rights reserved.                        |
+ |                                                                    |
+ | This work is published under the GNU AGPLv3 license with some      |
+ | permitted exceptions and without any warranty. For full license    |
+ | and copyright information, see https://civicrm.org/licensing       |
+ +--------------------------------------------------------------------+
+ */
+
+namespace Civi\Api4\Import;
+
+use Civi\Api4\Generic\DAOGetAction;
+use Civi\Api4\Generic\Result;
+
+class Import extends DAOGetAction {
+
+  use ImportProcessTrait;
+
+  /**
+   * @throws \CRM_Core_Exception
+   */
+  public function _run(Result $result): void {
+    $userJobID = (int) str_replace('Import_', '', $this->_entityName);
+    $where = $this->where;
+    $this->addWhere('_status', 'IN', ['new', 'valid']);
+    $this->getImportRows($result);
+    $parser = $this->getParser($userJobID);
+    foreach ($result as $row) {
+      $parser->import(array_values($row));
+    }
+    $parser->doPostImportActions();
+
+    // Re-fetch the validated result with updated messages.
+    $this->where = $where;
+    $this->addSelect('*');
+    parent::_run($result);
+  }
+
+}
diff --git a/ext/civiimport/Civi/Api4/Import/ImportProcessTrait.php b/ext/civiimport/Civi/Api4/Import/ImportProcessTrait.php
@@ -12,20 +12,73 @@
 
 namespace Civi\Api4\Import;
 
+use Civi\Api4\Generic\Result;
+use Civi\Api4\Import;
+use Civi\Api4\UserJob;
+
 /**
  * Code shared by Import Save/Update actions.
+ *
+ * @method getCheckPermissions()
  */
-trait ImportSaveTrait {
+trait ImportProcessTrait {
+
+  /**
+   * Get the parser for the import
+   *
+   * @return \CRM_Import_Parser|\CRM_Contribute_Import_Parser_Contribution
+   *
+   * @throws \CRM_Core_Exception
+   */
+  protected function getParser(int $userJobID) {
+    $userJob = UserJob::get($this->getCheckPermissions())
+      ->addWhere('id', '=', $userJobID)
+      ->addSelect('job_type')
+      ->execute()
+      ->first();
+    $parserClass = NULL;
+    foreach (\CRM_Core_BAO_UserJob::getTypes() as $userJobType) {
+      if ($userJob['job_type'] === $userJobType['id']) {
+        $parserClass = $userJobType['class'];
+      }
+    }
+    /** @var \CRM_Import_Parser|\CRM_Contribute_Import_Parser_Contribution $parser */
+    $parser = new $parserClass();
+    $parser->setUserJobID($userJobID);
+    $parser->init();
+    return $parser;
+  }
 
   /**
-   * @inheritDoc
+   * Get the selected import rows.
+   *
+   * @param \Civi\Api4\Generic\Result $result
+   *
+   * @throws \CRM_Core_Exception
    */
-  protected function write(array $items) {
-    $userJobID = str_replace('Import_', '', $this->_entityName);
-    foreach ($items as &$item) {
-      $item['_user_job_id'] = (int) $userJobID;
+  protected function getImportRows(Result $result): void {
+    $userJobID = (int) str_replace('Import_', '', $this->_entityName);
+    $this->addSelect('*');
+    $importFields = array_keys((array) Import::getFields($userJobID, $this->getCheckPermissions())
+      ->addSelect('name')
+      ->addWhere('name', 'NOT LIKE', '_%')
+      ->execute()
+      ->indexBy('name'));
+    $importFields[] = '_id';
+    $importFields[] = '_entity_id';
+    $this->setSelect($importFields);
+    parent::_run($result);
+    foreach ($result as &$row) {
+      if ($row['_entity_id']) {
+        // todo - how should we handle this? Skip, exception. At this case
+        // it is non ui accessible so this is good for now.
+        throw new \CRM_Core_Exception('Row already imported');
+      }
+      // Push ID to the end as the get has moved it to the front & order matters here.
+      $rowID = $row['_id'];
+      unset($row['_id'], $row['_entity_id']);
+      $row['_id'] = $rowID;
     }
-    return parent::write($items);
   }
 
 }
diff --git a/ext/civiimport/Civi/Api4/Import/Validate.php b/ext/civiimport/Civi/Api4/Import/Validate.php
@@ -12,10 +12,28 @@
 
 namespace Civi\Api4\Import;
 
-use Civi\Api4\Generic\DAOCreateAction;
+use Civi\Api4\Generic\DAOGetAction;
+use Civi\Api4\Generic\Result;
 
-class Import extends DAOCreateAction {
+class Validate extends DAOGetAction {
 
-  use ImportSaveTrait;
+  use ImportProcessTrait;
+
+  /**
+   * @throws \CRM_Core_Exception
+   */
+  public function _run(Result $result): void {
+    $userJobID = (int) str_replace('Import_', '', $this->_entityName);
+
+    $this->getImportRows($result);
+    $parser = $this->getParser($userJobID);
+    foreach ($result as $row) {
+      $parser->validateRow($row);
+    }
+
+    // Re-fetch the validated result with updated messages.
+    $this->addSelect('*');
+    parent::_run($result);
+  }
 
 }
diff --git a/ext/civiimport/Civi/Api4/Service/Spec/Provider/ImportSpecProvider.php b/ext/civiimport/Civi/Api4/Service/Spec/Provider/ImportSpecProvider.php
@@ -41,6 +41,7 @@ public function modifySpec(RequestSpec $spec): void {
       $field = new FieldSpec($column['name'], $spec->getEntity(), 'String');
       $field->setTitle(ts('Import field') . ':' . $column['label']);
       $field->setLabel($column['label']);
+      $field->setType('Field');
       $field->setReadonly($isInternalField);
       $field->setDescription(ts('Data being imported into the field.'));
       $field->setColumnName($column['name']);

diff --git a/ext/civiimport/tests/phpunit/CiviApiImportTest.php b/ext/civiimport/tests/phpunit/CiviApiImportTest.php
@@ -63,6 +63,13 @@ public function testApiActions():void {
           'dedupe_rule_id' => NULL,
           'dateFormats' => CRM_Core_Form_Date::DATE_yyyy_mm_dd,
         ],
+        'import_mappings' => [
+          ['name' => 'external_identifier'],
+          ['name' => 'total_amount'],
+          ['name' => 'receive_date'],
+          ['name' => 'financial_type_id'],
+          [],
+        ],
       ],
       'status_id:name' => 'draft',
       'job_type' => 'contribution_import',
@@ -82,11 +89,12 @@ public function testApiActions():void {
     ])->execute();
 
     $import = Import::get($userJobID)->setSelect(['external_identifier', 'amount_given', '_status'])->execute()->first();
+    $rowID = $import['_id'];
     $this->assertEquals('80', $import['amount_given']);
 
     Import::update($userJobID)->setValues([
       'amount_given' => NULL,
-      '_id' => $import['_id'],
+      '_id' => $rowID,
       '_status' => 'IMPORTED',
     ])->execute();
 
@@ -98,23 +106,35 @@ public function testApiActions():void {
         'external_identifier' => 999,
         'amount_given' => 9,
         '_status' => 'ERROR',
+        '_id' => $rowID,
       ],
     ])->execute();
 
-    $import = Import::get($userJobID)->setSelect(['external_identifier', 'amount_given', '_status'])->addWhere('_id', '>', $import['_id'])->execute()->first();
+    $import = Import::get($userJobID)->setSelect(['external_identifier', 'amount_given', '_status'])->addWhere('_id', '=', $rowID)->execute()->first();
     $this->assertEquals(9, $import['amount_given']);
 
     Import::save($userJobID)->setRecords([
       [
         'external_identifier' => 777,
-        '_id' => $import['_id'],
+        '_id' => $rowID,
         '_status' => 'ERROR',
       ],
     ])->execute();
 
-    $import = Import::get($userJobID)->setSelect(['external_identifier', 'amount_given', '_status'])->addWhere('_id', '=', $import['_id'])->execute()->first();
+    $import = Import::get($userJobID)->setSelect(['external_identifier', 'amount_given', '_status'])->addWhere('_id', '=', $rowID)->execute()->first();
     $this->assertEquals(777, $import['external_identifier']);
 
+    $validate = Import::validate($userJobID)->addWhere('_id', '=', $rowID)->setLimit(1)->execute()->first();
+    $this->assertEquals('Missing required fields: Contribution ID OR Invoice Reference OR Transaction ID OR Financial Type ID', $validate['_status_message']);
+    $this->assertEquals('ERROR', $validate['_status']);
+
+    Import::update($userJobID)->setValues(['financial_type' => 'Donation'])->addWhere('_id', '=', $rowID)->execute();
+    $validate = Import::validate($userJobID)->addWhere('_id', '=', $rowID)->setLimit(1)->execute()->first();
+    $this->assertEquals('', $validate['_status_message']);
+    $this->assertEquals('VALID', $validate['_status']);
+    $imported = Import::import($userJobID)->addWhere('_id', '=', $rowID)->setLimit(1)->execute()->first();
+    $this->assertEquals('ERROR', $imported['_status']);
+    $this->assertEquals('No matching Contact found', $imported['_status_message']);
   }
 
   /**