Merge pull request #26125 from MegaphoneJon/configurable-import-batch

Importer - Add system-setting for batch size
civicrm · May 2, 2023 · 4976021 · 4976021
2 parents 1721c18 + 77039ca
commit 4976021
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 1 deletion.
diff --git a/CRM/Admin/Form/Setting/Miscellaneous.php b/CRM/Admin/Form/Setting/Miscellaneous.php
@@ -42,6 +42,7 @@ class CRM_Admin_Form_Setting_Miscellaneous extends CRM_Admin_Form_Setting {
     'remote_profile_submissions' => CRM_Core_BAO_Setting::SYSTEM_PREFERENCES_NAME,
     'allow_alert_autodismissal' => CRM_Core_BAO_Setting::SYSTEM_PREFERENCES_NAME,
     'prevNextBackend' => CRM_Core_BAO_Setting::SEARCH_PREFERENCES_NAME,
+    'import_batch_size' => CRM_Core_BAO_Setting::SEARCH_PREFERENCES_NAME,
   ];
 
   /**
@@ -62,6 +63,7 @@ public function preProcess() {
       'recentItemsProviders',
       'dedupe_default_limit',
       'prevNextBackend',
+      'import_batch_size',
     ]);
   }
 

diff --git a/CRM/Import/Parser.php b/CRM/Import/Parser.php
@@ -672,7 +672,7 @@ public function queue(): void {
     $queue = Civi::queue('user_job_' . $this->getUserJobID(), ['type' => 'Sql', 'error' => 'abort', 'runner' => 'task', 'user_job_id' => $this->getUserJobID(), 'retry_limit' => 5]);
     UserJob::update(FALSE)->setValues(['queue_id.name' => 'user_job_' . $this->getUserJobID()])->addWhere('id', '=', $this->getUserJobID())->execute();
     $offset = 0;
-    $batchSize = 50;
+    $batchSize = Civi::settings()->get('import_batch_size');
     while ($totalRows > 0) {
       if ($totalRows < $batchSize) {
         $batchSize = $totalRows;

diff --git a/settings/Core.setting.php b/settings/Core.setting.php
@@ -1006,6 +1006,24 @@
       'callback' => 'CRM_Utils_Recent::getProviders',
     ],
   ],
+  'import_batch_size' => [
+    'name' => 'import_batch_size',
+    'type' => 'Integer',
+    'default' => 50,
+    'quick_form_type' => 'Element',
+    'html_type' => 'text',
+    'html_attributes' => [
+      'size' => 2,
+      'maxlength' => 3,
+    ],
+    'add' => '5.62',
+    'title' => ts('Import Batch Size'),
+    'is_domain' => 1,
+    'is_contact' => 0,
+    'description' => ts('Number of records to process at once during import.'),
+    'help_text' => ts('If your imports time out, reduce this number. You can increase it for better import performance on servers with longer timeouts.'),
+    'settings_pages' => 'misc',
+  ],
   'dedupe_default_limit' => [
     'group_name' => 'CiviCRM Preferences',
     'group' => 'core',