SearchKit - Add display of type entity

An entity display does not produce user-facing output, instead it writes to a SQL table
which can then be queried from SearchKit, the API, or other SQL-based tools like Drupal Views.

The new table is static; this includes a scheduled job to refresh it (disabled by default).

Civi/Api4/Generic/Traits/SavedSearchInspectorTrait.php

@@ -46,13 +46,14 @@ trait SavedSearchInspectorTrait {
 
   /**
    * If SavedSearch is supplied as a string, this will load it as an array
+   * @param int|null $id
+   * @throws UnauthorizedException
    * @throws \CRM_Core_Exception
-   * @throws \Civi\API\Exception\UnauthorizedException
    */
-  protected function loadSavedSearch() {
-    if (is_string($this->savedSearch)) {
+  protected function loadSavedSearch(int $id = NULL) {
+    if ($id || is_string($this->savedSearch)) {
       $this->savedSearch = SavedSearch::get(FALSE)
-        ->addWhere('name', '=', $this->savedSearch)
+        ->addWhere($id ? 'id' : 'name', '=', $id ?: $this->savedSearch)
         ->execute()->single();
     }
     if (is_array($this->savedSearch)) {
@@ -64,6 +65,8 @@ protected function loadSavedSearch() {
       ];
       $this->savedSearch['api_params'] += ['version' => 4, 'select' => [], 'where' => []];
     }
+    // Reset internal cached metadata
+    $this->_selectQuery = $this->_selectClause = $this->_searchEntityFields = NULL;
     $this->_apiParams = ($this->savedSearch['api_params'] ?? []) + ['select' => [], 'where' => []];
   }
 

Civi/Schema/Traits/OptionsSpecTrait.php

@@ -35,6 +35,11 @@ trait OptionsSpecTrait {
    */
   private $optionsCallback;
 
+  /**
+   * @var array
+   */
+  private $optionsCallbackParams = [];
+
   /**
    * @param array $values
    * @param array|bool $return
@@ -45,7 +50,7 @@ trait OptionsSpecTrait {
   public function getOptions($values = [], $return = TRUE, $checkPermissions = TRUE) {
     if (!isset($this->options)) {
       if ($this->optionsCallback) {
-        $this->options = ($this->optionsCallback)($this, $values, $return, $checkPermissions);
+        $this->options = ($this->optionsCallback)($this, $values, $return, $checkPermissions, $this->optionsCallbackParams);
       }
       else {
         $this->options = FALSE;
@@ -76,11 +81,15 @@ public function setSuffixes($suffixes) {
 
   /**
    * @param callable $callback
-   *
+   *   Function to be called, will receive the following arguments:
+   *   ($this, $values, $returnFormat, $checkPermissions, $params)
+   * @param array $params
+   *   Array of optional extra data; sent as 5th argument to the callback
    * @return $this
    */
-  public function setOptionsCallback($callback) {
+  public function setOptionsCallback($callback, array $params = []) {
     $this->optionsCallback = $callback;
+    $this->optionsCallbackParams = $params;
     return $this;
   }
 

ext/search_kit/Civi/Api4/Action/SKEntity/GetRefreshDate.php

@@ -0,0 +1,34 @@
+<?php
+
+namespace Civi\Api4\Action\SKEntity;
+
+use Civi\Api4\Generic\AbstractAction;
+use Civi\Api4\Generic\Result;
+
+/**
+ * Get the date the stored data was last refreshed for $ENTITY
+ *
+ * @package Civi\Api4\Action\SKEntity
+ */
+class GetRefreshDate extends AbstractAction {
+
+  /**
+   * @param \Civi\Api4\Generic\Result $result
+   * @throws \CRM_Core_Exception
+   */
+  public function _run(Result $result) {
+    [, $displayName] = explode('_', $this->getEntityName(), 2);
+    $tableName = 'civicrm_sk_' . \CRM_Utils_String::convertStringToSnakeCase($displayName);
+    $dbPath = explode('/', parse_url(CIVICRM_DSN, PHP_URL_PATH));
+    $dbName = end($dbPath);
+
+    $result[] = [
+      'refresh_date' => \CRM_Core_DAO::singleValueQuery("
+        SELECT UPDATE_TIME
+        FROM information_schema.tables
+        WHERE TABLE_SCHEMA = '$dbName'
+        AND TABLE_NAME = '$tableName'"),
+    ];
+  }
+
+}

ext/search_kit/Civi/Api4/Action/SKEntity/Refresh.php

@@ -0,0 +1,50 @@
+<?php
+
+namespace Civi\Api4\Action\SKEntity;
+
+use Civi\API\Request;
+use Civi\Api4\Generic\AbstractAction;
+use Civi\Api4\Generic\Result;
+use Civi\Api4\Query\Api4SelectQuery;
+
+/**
+ * Store the results of a SearchDisplay as a SQL table.
+ *
+ * For displays of type `entity` which save to a DB table
+ * rather than outputting anything to the user.
+ *
+ * @package Civi\Api4\Action\SKEntity
+ */
+class Refresh extends AbstractAction {
+
+  /**
+   * @param \Civi\Api4\Generic\Result $result
+   * @throws \CRM_Core_Exception
+   */
+  public function _run(Result $result) {
+    [, $displayName] = explode('_', $this->getEntityName(), 2);
+    $display = \Civi\Api4\SearchDisplay::get(FALSE)
+      ->setSelect(['settings', 'saved_search_id.api_entity', 'saved_search_id.api_params'])
+      ->addWhere('type', '=', 'entity')
+      ->addWhere('name', '=', $displayName)
+      ->execute()->single();
+
+    $apiParams = $display['saved_search_id.api_params'];
+    foreach ($display['settings']['sort'] ?? [] as $item) {
+      $apiParams['orderBy'][$item[0]] = $item[1];
+    }
+    $api = Request::create($display['saved_search_id.api_entity'], 'get', $apiParams);
+    $query = new Api4SelectQuery($api);
+    $query->forceSelectId = FALSE;
+    $select = $query->getSql();
+    $tableName = 'civicrm_sk_' . \CRM_Utils_String::convertStringToSnakeCase($displayName);
+    $columnSpecs = array_column($display['settings']['columns'], 'spec');
+    $columns = implode(', ', array_column($columnSpecs, 'name'));
+    \CRM_Core_DAO::executeQuery("TRUNCATE TABLE `$tableName`");
+    \CRM_Core_DAO::executeQuery("INSERT INTO `$tableName` ($columns) $select");
+    $result[] = [
+      'refresh_date' => \CRM_Core_DAO::singleValueQuery("SELECT NOW()"),
+    ];
+  }
+
+}

ext/search_kit/Civi/Api4/Event/Subscriber/SKEntitySubscriber.php

@@ -0,0 +1,230 @@
+<?php
+/*
+ +--------------------------------------------------------------------+
+ | Copyright CiviCRM LLC. All rights reserved.                        |
+ |                                                                    |
+ | This work is published under the GNU AGPLv3 license with some      |
+ | permitted exceptions and without any warranty. For full license    |
+ | and copyright information, see https://civicrm.org/licensing       |
+ +--------------------------------------------------------------------+
+ */
+
+namespace Civi\Api4\Event\Subscriber;
+
+use Civi\Api4\Generic\Traits\SavedSearchInspectorTrait;
+use Civi\Api4\Job;
+use Civi\Api4\SKEntity;
+use Civi\Api4\Utils\CoreUtil;
+use Civi\Core\Event\GenericHookEvent;
+use Civi\Core\Event\PostEvent;
+use Civi\Core\Event\PreEvent;
+use Civi\Core\Service\AutoService;
+use Symfony\Component\EventDispatcher\EventSubscriberInterface;
+
+/**
+ * Manages tables and API entities created from search displays of type "entity"
+ * @service
+ * @internal
+ */
+class SKEntitySubscriber extends AutoService implements EventSubscriberInterface {
+
+  use SavedSearchInspectorTrait;
+
+  /**
+   * @return array
+   */
+  public static function getSubscribedEvents(): array {
+    return [
+      'civi.api4.entityTypes' => 'on_civi_api4_entityTypes',
+      'hook_civicrm_pre' => 'onPreSaveDisplay',
+      'hook_civicrm_post' => 'onPostSaveDisplay',
+    ];
+  }
+
+  /**
+   * Register SearchDisplays of type 'entity'
+   *
+   * @param \Civi\Core\Event\GenericHookEvent $event
+   */
+  public static function on_civi_api4_entityTypes(GenericHookEvent $event): void {
+    // Can't use the API to fetch search displays because this hook is called when the API boots
+    foreach (_search_kit_get_entity_displays() as $display) {
+      $event->entities[$display['entityName']] = [
+        'name' => $display['entityName'],
+        'title' => $display['label'],
+        'title_plural' => $display['label'],
+        'description' => $display['settings']['description'] ?? NULL,
+        'primary_key' => ['_row'],
+        'type' => ['SavedSearch'],
+        'table_name' => $display['tableName'],
+        'class_args' => [$display['name']],
+        'label_field' => NULL,
+        'searchable' => 'secondary',
+        'class' => SKEntity::class,
+        'icon' => 'fa-search-plus',
+      ];
+    }
+  }
+
+  /**
+   * @param \Civi\Core\Event\PreEvent $event
+   */
+  public function onPreSaveDisplay(PreEvent $event): void {
+    if (!$this->applies($event)) {
+      return;
+    }
+    $oldName = $event->id ? \CRM_Core_DAO::getFieldValue('CRM_Search_DAO_SearchDisplay', $event->id) : NULL;
+    $newName = $event->params['name'] ?? $oldName;
+    $newSettings = $event->params['settings'] ?? NULL;
+    // No changes made, nothing to do
+    if (!$newSettings && $oldName === $newName && $event->action !== 'delete') {
+      return;
+    }
+    // Drop the old table if it exists
+    if ($oldName) {
+      \CRM_Core_BAO_SchemaHandler::dropTable('civicrm_sk_' . \CRM_Utils_String::convertStringToSnakeCase($oldName));
+    }
+    if ($event->action === 'delete') {
+      // Delete scheduled jobs when deleting entity
+      Job::delete(FALSE)
+        ->addWhere('api_entity', '=', 'SK_' . $oldName)
+        ->execute();
+      return;
+    }
+    // Build the new table
+    $savedSearchID = $event->params['saved_search_id'] ?? \CRM_Core_DAO::getFieldValue('CRM_Search_DAO_SearchDisplay', $event->id, 'saved_search_id');
+    $this->loadSavedSearch($savedSearchID);
+    $table = [
+      'name' => 'civicrm_sk_' . \CRM_Utils_String::convertStringToSnakeCase($newName),
+      'is_multiple' => FALSE,
+      'attributes' => 'ENGINE=InnoDB',
+      'fields' => [],
+    ];
+    // Primary key field
+    $table['fields'][] = [
+      'name' => '_row',
+      'type' => 'int unsigned',
+      'primary' => TRUE,
+      'required' => TRUE,
+      'attributes' => 'AUTO_INCREMENT',
+      'comment' => 'Row number',
+    ];
+    foreach ($newSettings['columns'] as &$column) {
+      $expr = $this->getSelectExpression($column['key']);
+      if (!$expr) {
+        continue;
+      }
+      $column['spec'] = $this->formatFieldSpec($column, $expr);
+      $table['fields'][] = $this->formatSQLSpec($column, $expr);
+    }
+    // Store new settings with added column spec
+    $event->params['settings'] = $newSettings;
+    $sql = \CRM_Core_BAO_SchemaHandler::buildTableSQL($table);
+    // do not i18n-rewrite
+    \CRM_Core_DAO::executeQuery($sql, [], TRUE, NULL, FALSE, FALSE);
+  }
+
+  /**
+   * @param array $column
+   * @param array{fields: array, expr: SqlExpression, dataType: string} $expr
+   * @return array
+   */
+  private function formatFieldSpec(array $column, array $expr): array {
+    // Strip the pseuoconstant suffix
+    [$name, $suffix] = array_pad(explode(':', $column['key']), 2, NULL);
+    // Sanitize the name
+    $name = \CRM_Utils_String::munge($name, '_', 255);
+    $spec = [
+      'name' => $name,
+      'data_type' => $expr['dataType'],
+      'suffixes' => $suffix ? ['id', $suffix] : NULL,
+      'options' => FALSE,
+    ];
+    if ($expr['expr']->getType() === 'SqlField') {
+      $field = \CRM_Utils_Array::first($expr['fields']);
+      $spec['fk_entity'] = $field['fk_entity'] ?? NULL;
+      $spec['original_field_name'] = $field['name'];
+      $spec['original_field_entity'] = $field['entity'];
+      if ($suffix) {
+        // Options will be looked up by SKEntitySpecProvider::getOptionsForSKEntityField
+        $spec['options'] = TRUE;
+      }
+    }
+    elseif ($expr['expr']->getType() === 'SqlFunction') {
+      if ($suffix) {
+        $spec['options'] = CoreUtil::formatOptionList($expr['expr']::getOptions(), $spec['suffixes']);
+      }
+    }
+    return $spec;
+  }
+
+  /**
+   * @param array $column
+   * @param array{fields: array, expr: SqlExpression, dataType: string} $expr
+   * @return array
+   */
+  private function formatSQLSpec(array $column, array $expr): array {
+    // Try to use the exact sql column type as the original field
+    $field = \CRM_Utils_Array::first($expr['fields']);
+    if (!empty($field['column_name']) && !empty($field['table_name'])) {
+      $columns = \CRM_Core_DAO::executeQuery("DESCRIBE `{$field['table_name']}`")
+        ->fetchMap('Field', 'Type');
+      $type = $columns[$field['column_name']] ?? NULL;
+    }
+    // If we can't get the exact data type from the column, take an educated guess
+    if (empty($type) ||
+      ($expr['expr']->getType() !== 'SqlField' && $field['data_type'] !== $expr['dataType'])
+    ) {
+      $map = [
+        'Array' => 'text',
+        'Boolean' => 'tinyint',
+        'Date' => 'date',
+        'Float' => 'double',
+        'Integer' => 'int',
+        'String' => 'text',
+        'Text' => 'text',
+        'Timestamp' => 'datetime',
+      ];
+      $type = $map[$expr['dataType']] ?? $type;
+    }
+    $defn = [
+      'name' => $column['spec']['name'],
+      'type' => $type,
+      // Adds an index to non-fk fields
+      'searchable' => TRUE,
+    ];
+    // Add FK indexes
+    if ($expr['expr']->getType() === 'SqlField' && !empty($field['fk_entity'])) {
+      $defn['fk_table_name'] = CoreUtil::getTableName($field['fk_entity']);
+      // FIXME look up fk_field_name from schema, don't assume it's always "id"
+      $defn['fk_field_name'] = 'id';
+      $defn['fk_attributes'] = ' ON DELETE SET NULL';
+    }
+    return $defn;
+  }
+
+  /**
+   * @param \Civi\Core\Event\PostEvent $event
+   */
+  public function onPostSaveDisplay(PostEvent $event): void {
+    if ($this->applies($event)) {
+      \CRM_Core_DAO_AllCoreTables::flush();
+      \Civi::cache('metadata')->clear();
+    }
+  }
+
+  /**
+   * Check if pre/post hook applies to a SearchDisplay type 'entity'
+   *
+   * @param \Civi\Core\Event\PreEvent|\Civi\Core\Event\PostEvent $event
+   * @return bool
+   */
+  private function applies(GenericHookEvent $event): bool {
+    if ($event->entity !== 'SearchDisplay') {
+      return FALSE;
+    }
+    $type = $event->params['type'] ?? $event->object->type ?? \CRM_Core_DAO::getFieldValue('CRM_Search_DAO_SearchDisplay', $event->id, 'type');
+    return $type === 'entity';
+  }
+
+}