Merge pull request #26583 from colemanw/setops2

APIv4 - Improve UNION field handling & add test coverage

Civi/Api4/Generic/Traits/SelectParamTrait.php

@@ -52,7 +52,7 @@ public function addSelect(string ...$fieldNames) {
    *
    * @throws \CRM_Core_Exception
    */
-  protected function expandSelectClauseWildcards() {
+  public function expandSelectClauseWildcards() {
     if (!$this->select) {
       $this->select = ['*'];
     }

Civi/Api4/Query/Api4EntitySetQuery.php

@@ -38,16 +38,20 @@ public function __construct($api) {
       // For non-aggregated queries, add a tracking id so the rows can be identified
       // for output-formatting purposes
       if (!$isAggregate) {
+        if (!$apiRequest->getSelect()) {
+          $apiRequest->addSelect('*');
+        }
         $apiRequest->addSelect($index . ' AS _api_set_index');
       }
-      $selectQuery = new Api4SelectQuery($apiRequest);
-      $selectQuery->forceSelectId = FALSE;
-      $selectQuery->getSql();
+      $apiRequest->expandSelectClauseWildcards();
+      $subQuery = new Api4SelectQuery($apiRequest);
+      $subQuery->forceSelectId = FALSE;
+      $subQuery->getSql();
       // Update field aliases of all subqueries to match the first query
       if ($index) {
-        $selectQuery->selectAliases = array_combine(array_keys($this->getSubquery()->selectAliases), $selectQuery->selectAliases);
+        $subQuery->selectAliases = array_combine(array_keys($this->getSubquery()->selectAliases), $subQuery->selectAliases);
       }
-      $this->subqueries[] = [$type, $selectQuery];
+      $this->subqueries[] = [$type, $subQuery];
     }
   }
 
@@ -93,37 +97,45 @@ protected function buildSelectClause() {
     // Add all subqueries to the FROM clause
     foreach ($this->subqueries as $index => $set) {
       [$type, $selectQuery] = $set;
-
       $this->query->setOp($type, [$selectQuery->getQuery()]);
+    }
+    // Build apiFieldSpec from the select clause of the first query
+    foreach ($this->getSubquery()->selectAliases as $alias => $sql) {
       // If this outer query uses the default of SELECT * then effectively we are selecting
       // all the fields of the first subquery
-      if (!$index && !$select) {
-        $this->selectAliases = $selectQuery->selectAliases;
-        $this->apiFieldSpec = $selectQuery->apiFieldSpec;
+      if (!$select) {
+        $this->selectAliases[$alias] = $alias;
       }
+      $expr = SqlExpression::convert($sql);
+      $field = $expr->getType() === 'SqlField' ? $this->getSubquery()->getField($expr->getFields()[0]) : NULL;
+      $this->addSpecField($alias, [
+        'sql_name' => "`$alias`",
+        'entity' => $field['entity'] ?? NULL,
+        'data_type' => $field['data_type'] ?? $expr::getDataType(),
+      ]);
     }
     // Parse select clause if not using default of *
     foreach ($select as $item) {
       $expr = SqlExpression::convert($item, TRUE);
-      foreach ($expr->getFields() as $fieldName) {
-        $field = $this->getField($fieldName);
-        $this->apiFieldSpec[$fieldName] = $field;
-      }
       $alias = $expr->getAlias();
       $this->selectAliases[$alias] = $expr->getExpr();
       $this->query->select($expr->render($this) . " AS `$alias`");
     }
   }
 
-  public function getField($expr, $strict = FALSE) {
+  /**
+   * @param string $expr
+   * @return array|null
+   */
+  public function getField($expr) {
     $col = strpos($expr, ':');
     $fieldName = $col ? substr($expr, 0, $col) : $expr;
-    return $this->apiFieldSpec[$fieldName] ?? $this->getSubquery()->getField($expr, $strict);
+    return $this->apiFieldSpec[$fieldName] ?? NULL;
   }
 
   protected function buildWhereClause() {
     foreach ($this->getWhere() as $clause) {
-      $sql = $this->treeWalkClauses($clause, 'HAVING');
+      $sql = $this->treeWalkClauses($clause, 'WHERE');
       if ($sql) {
         $this->query->where($sql);
       }
@@ -152,8 +164,13 @@ protected function buildOrderBy() {
       if ($dir !== 'ASC' && $dir !== 'DESC') {
         throw new \CRM_Core_Exception("Invalid sort direction. Cannot order by $item $dir");
       }
-      $expr = $this->getExpression($item);
-      $column = $this->renderExpr($expr);
+      if (!empty($this->selectAliases[$item])) {
+        $column = '`' . $item . '`';
+      }
+      else {
+        $expr = $this->getExpression($item);
+        $column = $this->renderExpr($expr);
+      }
       $this->query->orderBy("$column $dir");
     }
   }

Civi/Api4/Query/Api4Query.php

@@ -146,6 +146,9 @@ public function addSpecField($path, $field) {
       return;
     }
     $this->apiFieldSpec[$path] = $field + [
+      'name' => $path,
+      'type' => 'Extra',
+      'entity' => NULL,
       'implicit_join' => NULL,
       'explicit_join' => NULL,
     ];
@@ -296,8 +299,10 @@ public function composeClause(array $clause, string $type, int $depth) {
       $fieldAlias = $expr->render($this);
       if (is_string($value)) {
         $valExpr = $this->getExpression($value);
+        // Format string input
         if ($expr->getType() === 'SqlField' && $valExpr->getType() === 'SqlString') {
-          $value = $valExpr->getExpr();
+          // Strip surrounding quotes
+          $value = substr($valExpr->getExpr(), 1, -1);
           FormattingUtil::formatInputValue($value, $fieldName, $this->apiFieldSpec[$fieldName], $this->entityValues, $operator);
           return $this->createSQLClause($fieldAlias, $operator, $value, $this->apiFieldSpec[$fieldName], $depth);
         }

Civi/Api4/Query/SqlBool.php

@@ -0,0 +1,32 @@
+<?php
+/*
+ +--------------------------------------------------------------------+
+ | Copyright CiviCRM LLC. All rights reserved.                        |
+ |                                                                    |
+ | This work is published under the GNU AGPLv3 license with some      |
+ | permitted exceptions and without any warranty. For full license    |
+ | and copyright information, see https://civicrm.org/licensing       |
+ +--------------------------------------------------------------------+
+ */
+
+namespace Civi\Api4\Query;
+
+/**
+ * Boolean sql expression
+ */
+class SqlBool extends SqlExpression {
+
+  protected static $dataType = 'Boolean';
+
+  protected function initialize() {
+  }
+
+  public function render(Api4Query $query): string {
+    return $this->expr === 'TRUE' ? '1' : '0';
+  }
+
+  public static function getTitle(): string {
+    return ts('Boolean');
+  }
+
+}

Civi/Api4/Query/SqlExpression.php

@@ -21,7 +21,8 @@
 abstract class SqlExpression {
 
   /**
-   * @var array
+   * Field names used in this expression
+   * @var string[]
    */
   protected $fields = [];
 
@@ -102,6 +103,9 @@ public static function convert(string $expression, $parseAlias = FALSE, $mustBe
     elseif ($expr === 'NULL') {
       $className = 'SqlNull';
     }
+    elseif ($expr === 'TRUE' || $expr === 'FALSE') {
+      $className = 'SqlBool';
+    }
     elseif ($expr === '*') {
       $className = 'SqlWild';
     }

Civi/Api4/Query/SqlString.php

@@ -31,6 +31,13 @@ public function render(Api4Query $query): string {
     return '"' . \CRM_Core_DAO::escapeString($this->expr) . '"';
   }
 
+  /**
+   * @return string
+   */
+  public function getExpr(): string {
+    return '"' . $this->expr . '"';
+  }
+
   public static function getTitle(): string {
     return ts('Text');
   }

tests/phpunit/api/v4/Action/EntitySetUnionTest.php

@@ -20,6 +20,7 @@
 namespace api\v4\Action;
 
 use api\v4\Api4TestBase;
+use Civi\Api4\ContactType;
 use Civi\Api4\EntitySet;
 use Civi\Api4\Group;
 use Civi\Api4\Relationship;
@@ -114,7 +115,80 @@ public function testGroupByUnionSet(): void {
     $this->assertEquals(2, $result[1]['count']);
     $this->assertEquals(2, $result[2]['count']);
     $this->assertEquals(1, $result[3]['count']);
+  }
+
+  public function testUnionWithSelectAndOrderBy(): void {
+    $contacts = $this->saveTestRecords('Contact', ['records' => 4])->column('id');
+    $relationships = $this->saveTestRecords('Relationship', [
+      'records' => [
+        ['contact_id_a' => $contacts[0], 'contact_id_b' => $contacts[1]],
+        ['contact_id_a' => $contacts[1], 'contact_id_b' => $contacts[2]],
+        ['contact_id_a' => $contacts[1], 'contact_id_b' => $contacts[3]],
+      ],
+    ]);
 
+    $result = EntitySet::get(FALSE)
+      ->addSelect('contact_id_b', 'UPPER(direction) AS DIR')
+      ->addSet('UNION ALL', Relationship::get()
+        ->addSelect('id', 'contact_id_a', 'contact_id_b', '"a_b" AS direction')
+        ->addWhere('id', 'IN', $relationships->column('id'))
+      )
+      ->addSet('UNION ALL', Relationship::get()
+        ->addSelect('id', 'contact_id_b', 'contact_id_a', '"b_a" AS direction')
+        ->addWhere('id', 'IN', $relationships->column('id'))
+      )
+      ->addWhere('contact_id_a', '=', $contacts[1])
+      ->addOrderBy('direction')
+      ->addOrderBy('id')
+      ->execute();
+
+    $this->assertCount(3, $result);
+    $this->assertEquals('A_B', $result[0]['DIR']);
+    $this->assertEquals('A_B', $result[1]['DIR']);
+    $this->assertEquals('B_A', $result[2]['DIR']);
+    $this->assertEquals($contacts[2], $result[0]['contact_id_b']);
+    $this->assertEquals($contacts[3], $result[1]['contact_id_b']);
+    $this->assertEquals($contacts[0], $result[2]['contact_id_b']);
+  }
+
+  public function testUnionWithSelectStar() {
+    $subType = $this->createTestRecord('ContactType', [
+      'parent_id:name' => 'Household',
+      'name' => uniqid('HH1'),
+    ]);
+    $result = EntitySet::get(FALSE)
+      ->addSelect('name', 'label', 'parent_id:name')
+      ->addSet('UNION ALL', ContactType::get()
+        ->addWhere('name', '=', 'Household')
+      )
+      ->addSet('UNION ALL', ContactType::get()
+        ->addWhere('id', '=', $subType['id'])
+      )
+      ->addOrderBy('id')
+      ->execute();
+    $this->assertCount(2, $result);
+    $this->assertEquals('Household', $result[1]['parent_id:name']);
+    $this->assertEquals('Household', $result[0]['name']);
+
+    $result = EntitySet::get(FALSE)
+      ->addSelect('id', 'name', 'label', 'parent_id:name', 'is_parent')
+      ->addSet('UNION ALL', ContactType::get()
+        ->addSelect('*', 'TRUE AS is_parent')
+        ->addWhere('name', '=', 'Household')
+      )
+      ->addSet('UNION ALL', ContactType::get()
+        ->addSelect('*', 'FALSE AS is_parent')
+        ->addWhere('id', '=', $subType['id'])
+      )
+      ->addOrderBy('is_parent')
+      ->execute();
+
+    $this->assertCount(2, $result);
+    $this->assertEquals($subType['id'], $result[0]['id']);
+    $this->assertIsInt($result[1]['id']);
+    $this->assertFalse($result[0]['is_parent']);
+    $this->assertEquals('Household', $result[0]['parent_id:name']);
+    $this->assertEquals('Household', $result[1]['name']);
   }
 
 }