Api4 - Support wildcard * in select clause

Civi/Api4/Generic/AbstractAction.php

@@ -383,12 +383,13 @@ public function getPermissions() {
   /**
    * Returns schema fields for this entity & action.
    *
-   * Here we bypass the api wrapper and execute the getFields action directly.
+   * Here we bypass the api wrapper and run the getFields action directly.
    * This is because we DON'T want the wrapper to check permissions as this is an internal op,
    * but we DO want permissions to be checked inside the getFields request so e.g. the api_key
    * field can be conditionally included.
    * @see \Civi\Api4\Action\Contact\GetFields
    *
+   * @throws \API_Exception
    * @return array
    */
   public function entityFields() {

Civi/Api4/Generic/AbstractGetAction.php

@@ -21,6 +21,8 @@
 
 namespace Civi\Api4\Generic;
 
+use Civi\Api4\Utils\SelectUtil;
+
 /**
  * Base class for all "Get" api actions.
  *
@@ -33,7 +35,10 @@
 abstract class AbstractGetAction extends AbstractQueryAction {
 
   /**
-   * Fields to return. Defaults to all fields.
+   * Fields to return. Defaults to all fields ["*"].
+   *
+   * Use the * wildcard by itself to select all available fields, or use it to match similarly-named fields.
+   * E.g. "is_*" will match fields named is_primary, is_active, etc.
    *
    * Set to ["row_count"] to return only the number of items found.
    *
@@ -70,6 +75,20 @@ protected function setDefaultWhereClause() {
     }
   }
 
+  /**
+   * Adds all fields matched by the * wildcard
+   *
+   * @throws \API_Exception
+   */
+  protected function expandSelectClauseWildcards() {
+    foreach ($this->select as $item) {
+      if (strpos($item, '*') !== FALSE && strpos($item, '.') === FALSE) {
+        $this->select = array_diff($this->select, [$item]);
+        $this->select = array_unique(array_merge($this->select, SelectUtil::getMatchingFields($item, array_column($this->entityFields(), 'name'))));
+      }
+    }
+  }
+
   /**
    * Helper to parse the WHERE param for getRecords to perform simple pre-filtering.
    *

Civi/Api4/Generic/BasicGetAction.php

@@ -57,6 +57,7 @@ public function __construct($entityName, $actionName, $getter = NULL) {
    */
   public function _run(Result $result) {
     $this->setDefaultWhereClause();
+    $this->expandSelectClauseWildcards();
     $values = $this->getRecords();
     $result->exchangeArray($this->queryArray($values));
   }

Civi/Api4/Generic/DAOGetAction.php

@@ -31,8 +31,20 @@
 class DAOGetAction extends AbstractGetAction {
   use Traits\DAOActionTrait;
 
+  /**
+   * Fields to return. Defaults to all non-custom fields ["*"].
+   *
+   * Use the dot notation to perform joins in the select clause, e.g. selecting ["*", "contact.*"] from Email.get
+   * will select all fields for the email + all fields for the related contact.
+   *
+   * @var array
+   * @inheritDoc
+   */
+  protected $select = [];
+
   public function _run(Result $result) {
     $this->setDefaultWhereClause();
+    $this->expandSelectClauseWildcards();
     $result->exchangeArray($this->getObjects());
   }
 

Civi/Api4/Query/Api4SelectQuery.php

@@ -18,6 +18,7 @@
 use Civi\Api4\Service\Schema\Joinable\Joinable;
 use Civi\Api4\Utils\FormattingUtil;
 use Civi\Api4\Utils\CoreUtil;
+use Civi\Api4\Utils\SelectUtil;
 use CRM_Core_DAO_AllCoreTables as AllCoreTables;
 use CRM_Utils_Array as UtilsArray;
 
@@ -161,23 +162,16 @@ protected function addJoins() {
    * @throws \Civi\API\Exception\UnauthorizedException
    */
   protected function buildSelectFields() {
-    $return_all_fields = (empty($this->select) || !is_array($this->select));
-    $return = $return_all_fields ? $this->entityFieldNames : $this->select;
-    if ($return_all_fields || in_array('custom', $this->select)) {
-      foreach (array_keys($this->apiFieldSpec) as $fieldName) {
-        if (strpos($fieldName, 'custom_') === 0) {
-          $return[] = $fieldName;
-        }
-      }
-    }
+    $selectAll = (empty($this->select) || in_array('*', $this->select));
+    $select = $selectAll ? $this->entityFieldNames : $this->select;
 
     // Always select the ID if the table has one.
     if (array_key_exists('id', $this->apiFieldSpec) || strstr($this->entity, 'Custom_')) {
       $this->selectFields[self::MAIN_TABLE_ALIAS . ".id"] = "id";
     }
 
     // core return fields
-    foreach ($return as $fieldName) {
+    foreach ($select as $fieldName) {
       $field = $this->getField($fieldName);
       if (strpos($fieldName, '.') && !empty($this->fkSelectAliases[$fieldName]) && !array_filter($this->getPathJoinTypes($fieldName))) {
         $this->selectFields[$this->fkSelectAliases[$fieldName]] = $fieldName;
@@ -338,6 +332,14 @@ protected function joinFK($key) {
     /** @var \Civi\Api4\Service\Schema\Joinable\Joinable $lastLink */
     $lastLink = array_pop($joinPath);
 
+    $isWild = strpos($field, '*') !== FALSE;
+    if ($isWild) {
+      if (!in_array($key, $this->select)) {
+        throw new \API_Exception('Wildcards can only be used in the SELECT clause.');
+      }
+      $this->select = array_diff($this->select, [$key]);
+    }
+
     // Cache field info for retrieval by $this->getField()
     $prefix = array_pop($pathArray) . '.';
     if (!isset($this->apiFieldSpec[$prefix . $field])) {
@@ -351,19 +353,29 @@ protected function joinFK($key) {
       }
     }
 
-    if (!$lastLink->getField($field)) {
+    if (!$isWild && !$lastLink->getField($field)) {
       throw new \API_Exception('Invalid join');
     }
 
-    // custom groups use aliases for field names
-    if ($lastLink instanceof CustomGroupJoinable) {
-      $field = $lastLink->getSqlColumn($field);
+    $fields = $isWild ? [] : [$field];
+    // Expand wildcard and add matching fields to $this->select
+    if ($isWild) {
+      $fields = SelectUtil::getMatchingFields($field, $lastLink->getEntityFieldNames());
+      foreach ($fields as $field) {
+        $this->select[] = $pathString . '.' . $field;
+      }
+      $this->select = array_unique($this->select);
     }
-    // Check Permission on field.
-    if ($this->checkPermissions && !empty($this->apiFieldSpec[$prefix . $field]['permission']) && !\CRM_Core_Permission::check($this->apiFieldSpec[$prefix . $field]['permission'])) {
-      return;
+
+    foreach ($fields as $field) {
+      // custom groups use aliases for field names
+      $col = ($lastLink instanceof CustomGroupJoinable) ? $lastLink->getSqlColumn($field) : $field;
+      // Check Permission on field.
+      if ($this->checkPermissions && !empty($this->apiFieldSpec[$prefix . $field]['permission']) && !\CRM_Core_Permission::check($this->apiFieldSpec[$prefix . $field]['permission'])) {
+        return;
+      }
+      $this->fkSelectAliases[$pathString . '.' . $field] = sprintf('%s.%s', $lastLink->getAlias(), $col);
     }
-    $this->fkSelectAliases[$key] = sprintf('%s.%s', $lastLink->getAlias(), $field);
   }
 
   /**

Civi/Api4/Service/Schema/Joinable/CustomGroupJoinable.php

@@ -86,6 +86,9 @@ public function getField($fieldName) {
    * @return string
    */
   public function getSqlColumn($fieldName) {
+    if (strpos($fieldName, '.') !== FALSE) {
+      $fieldName = substr($fieldName, 1 + strrpos($fieldName, '.'));
+    }
     return $this->columns[$fieldName];
   }
 

Civi/Api4/Service/Schema/Joinable/Joinable.php

@@ -281,6 +281,17 @@ public function getEntityFields() {
     return $this->entityFields;
   }
 
+  /**
+   * @return array
+   */
+  public function getEntityFieldNames() {
+    $fieldNames = [];
+    foreach ($this->getEntityFields() as $fieldSpec) {
+      $fieldNames[] = $fieldSpec->getName();
+    }
+    return $fieldNames;
+  }
+
   /**
    * @return \Civi\Api4\Service\Spec\FieldSpec|NULL
    */

Civi/Api4/Utils/SelectUtil.php

@@ -0,0 +1,60 @@
+<?php
+
+/*
+ +--------------------------------------------------------------------+
+ | Copyright CiviCRM LLC. All rights reserved.                        |
+ |                                                                    |
+ | This work is published under the GNU AGPLv3 license with some      |
+ | permitted exceptions and without any warranty. For full license    |
+ | and copyright information, see https://civicrm.org/licensing       |
+ +--------------------------------------------------------------------+
+ */
+
+/**
+ *
+ * @package CRM
+ * @copyright CiviCRM LLC https://civicrm.org/licensing
+ * $Id$
+ *
+ */
+
+
+namespace Civi\Api4\Utils;
+
+class SelectUtil {
+
+  /**
+   * Checks if a field is in the Select array or matches a wildcard pattern in the Select array
+   *
+   * @param string $field
+   * @param array $selects
+   * @return bool
+   */
+  public static function isFieldSelected($field, $selects) {
+    if (in_array($field, $selects) || (in_array('*', $selects) && strpos($field, '.') === FALSE)) {
+      return TRUE;
+    }
+    foreach ($selects as $item) {
+      if (strpos($item, '*') !== FALSE && self::getMatchingFields($item, [$field])) {
+        return TRUE;
+      }
+    }
+    return FALSE;
+  }
+
+  /**
+   * @param string $pattern
+   * @param array $fieldNames
+   * @return array
+   */
+  public static function getMatchingFields($pattern, $fieldNames) {
+    if ($pattern === '*') {
+      return $fieldNames;
+    }
+    $pattern = '/^' . str_replace('\*', '.*', preg_quote($pattern, '/')) . '$/';
+    return array_values(array_filter($fieldNames, function($field) use ($pattern) {
+      return preg_match($pattern, $field);
+    }));
+  }
+
+}

ang/api4Explorer/Explorer.html

@@ -51,7 +51,7 @@ <h1 crm-page-title>
           </div>
           <div class="api4-input form-inline" ng-mouseenter="help('select', availableParams.select)" ng-mouseleave="help()" ng-if="availableParams.select && !isSelectRowCount()">
             <label for="api4-param-select">select<span class="crm-marker" ng-if="availableParams.select.required"> *</span></label>
-            <input class="collapsible-optgroups form-control" ng-list crm-ui-select="{data: fieldsAndJoins, multiple: true}" id="api4-param-select" ng-model="params.select" style="width: 85%;"/>
+            <input class="collapsible-optgroups form-control" ng-list crm-ui-select="{data: selectFieldsAndJoins, multiple: true}" placeholder="*" id="api4-param-select" ng-model="params.select" style="width: 85%;"/>
           </div>
           <div class="api4-input form-inline" ng-mouseenter="help('fields', availableParams.fields)" ng-mouseleave="help()"ng-if="availableParams.fields">
             <label for="api4-param-fields">fields<span class="crm-marker" ng-if="availableParams.fields.required"> *</span></label>

ang/api4Explorer/Explorer.js

@@ -26,6 +26,7 @@
     $scope.actions = actions;
     $scope.fields = [];
     $scope.fieldsAndJoins = [];
+    $scope.selectFieldsAndJoins = [];
     $scope.availableParams = {};
     $scope.params = {};
     $scope.index = '';
@@ -113,16 +114,17 @@
       return fields;
     }
 
-    function addJoins(fieldList) {
+    function addJoins(fieldList, addWildcard) {
       var fields = _.cloneDeep(fieldList),
         fks = _.findWhere(links, {entity: $scope.entity}) || {};
       _.each(fks.links, function(link) {
-        var linkFields = entityFields(link.entity);
+        var linkFields = _.cloneDeep(entityFields(link.entity)),
+          wildCard = addWildcard ? [{id: link.alias + '.*', text: link.alias + '.*', 'description': 'All core ' + link.entity + ' fields'}] : [];
         if (linkFields) {
           fields.push({
             text: link.alias,
             description: 'Join to ' + link.entity,
-            children: formatForSelect2(linkFields, [], 'name', ['description'], link.alias + '.')
+            children: wildCard.concat(formatForSelect2(linkFields, [], 'name', ['description'], link.alias + '.'))
           });
         }
       });
@@ -261,7 +263,8 @@
 
     function selectAction() {
       $scope.action = $routeParams.api4action;
-      $scope.fieldsAndJoins = [];
+      $scope.fieldsAndJoins.length = 0;
+      $scope.selectFieldsAndJoins.length = 0;
       if (!actions.length) {
         formatForSelect2(getEntity().actions, actions, 'name', ['description', 'params']);
       }
@@ -270,9 +273,12 @@
         $scope.fields = getFieldList($scope.action);
         if (_.contains(['get', 'update', 'delete', 'replace'], $scope.action)) {
           $scope.fieldsAndJoins = addJoins($scope.fields);
+          $scope.selectFieldsAndJoins = addJoins($scope.fields, true);
         } else {
           $scope.fieldsAndJoins = $scope.fields;
+          $scope.selectFieldsAndJoins = _.cloneDeep($scope.fields);
         }
+        $scope.selectFieldsAndJoins.unshift({id: '*', text: '*', 'description': 'All core ' + $scope.entity + ' fields'});
         _.each(actionInfo.params, function (param, name) {
           var format,
             defaultVal = _.cloneDeep(param.default);

api/api.php

@@ -63,7 +63,7 @@ function civicrm_api4(string $entity, string $action, array $params = [], $index
   $removeIndexField = FALSE;
 
   // If index field is not part of the select query, we add it here and remove it below
-  if ($indexField && !empty($params['select']) && is_array($params['select']) && !in_array($indexField, $params['select'])) {
+  if ($indexField && !empty($params['select']) && is_array($params['select']) && !\Civi\Api4\Utils\SelectUtil::isFieldSelected($indexField, $params['select'])) {
     $params['select'][] = $indexField;
     $removeIndexField = TRUE;
   }

tests/phpunit/api/v4/Action/GetFromArrayTest.php

@@ -61,7 +61,7 @@ public function testArrayGetWithSort() {
   public function testArrayGetWithSelect() {
     $result = MockArrayEntity::get()
       ->addSelect('field1')
-      ->addSelect('field3')
+      ->addSelect('f*3')
       ->setLimit(4)
       ->execute();
     $this->assertEquals([

tests/phpunit/api/v4/Entity/ContactJoinTest.php

@@ -56,7 +56,7 @@ public function testContactJoin() {
     foreach ($entitiesToTest as $entity) {
       $results = civicrm_api4($entity, 'get', [
         'where' => [['contact_id', '=', $contact['id']]],
-        'select' => ['contact.display_name', 'contact.id'],
+        'select' => ['contact.*_name', 'contact.id'],
       ]);
       foreach ($results as $result) {
         $this->assertEquals($contact['id'], $result['contact.id']);

tests/phpunit/api/v4/Mock/Api4/MockArrayEntity.php

@@ -34,7 +34,14 @@ class MockArrayEntity extends Generic\AbstractEntity {
 
   public static function getFields() {
     return new BasicGetFieldsAction(static::class, __FUNCTION__, function() {
-      return [];
+      return [
+        ['name' => 'field1'],
+        ['name' => 'field2'],
+        ['name' => 'field3'],
+        ['name' => 'field4'],
+        ['name' => 'field5'],
+        ['name' => 'field6'],
+      ];
     });
   }
 

tests/phpunit/api/v4/Query/Api4SelectQueryComplexJoinTest.php

@@ -49,7 +49,7 @@ public function testWithComplexRelatedEntitySelect() {
     $query = new Api4SelectQuery('Contact', FALSE, civicrm_api4('Contact', 'getFields', ['includeCustom' => FALSE, 'checkPermissions' => FALSE, 'action' => 'get'], 'name'));
     $query->select[] = 'id';
     $query->select[] = 'display_name';
-    $query->select[] = 'phones.phone';
+    $query->select[] = 'phones.*_id';
     $query->select[] = 'emails.email';
     $query->select[] = 'emails.location_type.name';
     $query->select[] = 'created_activities.contact_id';
@@ -75,6 +75,11 @@ public function testWithComplexRelatedEntitySelect() {
     $this->assertArrayHasKey('activity_type', $firstActivity);
     $activityType = $firstActivity['activity_type'];
     $this->assertArrayHasKey('name', $activityType);
+
+    $this->assertArrayHasKey('name', $firstResult['emails'][0]['location_type']);
+    $this->assertArrayHasKey('location_type_id', $firstResult['phones'][0]);
+    $this->assertArrayHasKey('id', $firstResult['phones'][0]);
+    $this->assertArrayNotHasKey('phone', $firstResult['phones'][0]);
   }
 
   public function testWithSelectOfOrphanDeepValues() {