Align most of the tokens in the token processor handling with the legacy handling

CRM/Utils/Token.php

@@ -732,22 +732,12 @@ public static function getContactTokenReplacement(
       $value = "cs={$cs}";
     }
     else {
-      $value = CRM_Utils_Array::retrieveValueRecursive($contact, $token);
+      $value = (array) CRM_Utils_Array::retrieveValueRecursive($contact, $token);
 
-      // FIXME: for some pseudoconstants we get array ( 0 => id, 1 => label )
-      if (is_array($value)) {
-        $value = $value[1];
-      }
-      // Convert pseudoconstants using metadata
-      elseif ($value && is_numeric($value)) {
-        $allFields = CRM_Contact_BAO_Contact::exportableFields('All');
-        if (!empty($allFields[$token]['pseudoconstant'])) {
-          $value = CRM_Core_PseudoConstant::getLabel('CRM_Contact_BAO_Contact', $token, $value);
-        }
-      }
-      elseif ($value && CRM_Utils_String::endsWith($token, '_date')) {
-        $value = CRM_Utils_Date::customFormat($value);
+      foreach ($value as $index => $item) {
+        $value[$index] = self::convertPseudoConstantsUsingMetadata($value[$index], $token);
       }
+      $value = implode(', ', $value);
     }
 
     if (!$html) {
@@ -1897,4 +1887,24 @@ public static function formatTokensForDisplay($tokens) {
     return $output;
   }
 
+  /**
+   * @param $value
+   * @param $token
+   *
+   * @return bool|int|mixed|string|null
+   */
+  protected static function convertPseudoConstantsUsingMetadata($value, $token) {
+    // Convert pseudoconstants using metadata
+    if ($value && is_numeric($value)) {
+      $allFields = CRM_Contact_BAO_Contact::exportableFields('All');
+      if (!empty($allFields[$token]['pseudoconstant'])) {
+        $value = CRM_Core_PseudoConstant::getLabel('CRM_Contact_BAO_Contact', $token, $value);
+      }
+    }
+    elseif ($value && CRM_Utils_String::endsWith($token, '_date')) {
+      $value = CRM_Utils_Date::customFormat($value);
+    }
+    return $value;
+  }
+
 }

Civi/Token/TokenCompatSubscriber.php

@@ -47,6 +47,8 @@ public function onEvaluate(TokenValueEvent $e) {
     $e->getTokenProcessor()->context['hookTokenCategories'] = $categories;
 
     $messageTokens = $e->getTokenProcessor()->getMessageTokens();
+    $returnProperties = array_fill_keys($messageTokens['contact'] ?? [], 1);
+    $returnProperties = array_merge(\CRM_Contact_BAO_Query::defaultReturnProperties(), $returnProperties);
 
     foreach ($e->getRows() as $row) {
       if (empty($row->context['contactId'])) {
@@ -58,23 +60,24 @@ public function onEvaluate(TokenValueEvent $e) {
         $params = [
           ['contact_id', '=', $contactId, 0, 0],
         ];
-        [$contact] = \CRM_Contact_BAO_Query::apiQuery($params);
+        [$contact] = \CRM_Contact_BAO_Query::apiQuery($params, $returnProperties ?? NULL);
         //CRM-4524
         $contact = reset($contact);
+        // Test cover for greeting in CRM_Core_BAO_ActionScheduleTest::testMailer
+        $contact['email_greeting'] = $contact['email_greeting_display'] ?? '';
+        $contact['postal_greeting'] = $contact['postal_greeting_display'] ?? '';
+        $contact['addressee'] = $contact['address_display'] ?? '';
         if (!$contact || is_a($contact, 'CRM_Core_Error')) {
           // FIXME: Need to differentiate errors which kill the batch vs the individual row.
-          \Civi::log()->debug("Failed to generate token data. Invalid contact ID: " . $row->context['contactId']);
+          \Civi::log()->debug('Failed to generate token data. Invalid contact ID: ' . $row->context['contactId']);
           continue;
         }
 
         //update value of custom field token
         if (!empty($messageTokens['contact'])) {
           foreach ($messageTokens['contact'] as $token) {
             if (\CRM_Core_BAO_CustomField::getKeyID($token)) {
-              $contact[$token] = civicrm_api3('Contact', 'getvalue', [
-                'return' => $token,
-                'id' => $contactId,
-              ]);
+              $contact[$token] = \CRM_Core_BAO_CustomField::displayValue($contact[$token], \CRM_Core_BAO_CustomField::getKeyID($token));
             }
           }
         }

tests/phpunit/CRM/Core/BAO/MessageTemplateTest.php

@@ -2,6 +2,7 @@
 
 use Civi\Api4\Address;
 use Civi\Api4\Contact;
+use Civi\Token\TokenProcessor;
 
 /**
  * Class CRM_Core_BAO_MessageTemplateTest
@@ -17,7 +18,7 @@ class CRM_Core_BAO_MessageTemplateTest extends CiviUnitTestCase {
    * @throws \CRM_Core_Exception
    */
   public function tearDown():void {
-    $this->quickCleanup(['civicrm_address', 'civicrm_phone', 'civicrm_im', 'civicrm_website', 'civicrm_openid', 'civicrm_email']);
+    $this->quickCleanup(['civicrm_address', 'civicrm_phone', 'civicrm_im', 'civicrm_website', 'civicrm_openid', 'civicrm_email'], TRUE);
     parent::tearDown();
   }
 
@@ -141,13 +142,7 @@ public function testRenderMessageTemplateIgnoreSmarty(): void {
   public function testContactTokens(): void {
     $this->createCustomGroupWithFieldsOfAllTypes([]);
     $tokenData = $this->getAllContactTokens();
-    $this->callAPISuccess('Contact', 'create', $tokenData);
-    $address = $this->callAPISuccess('Address', 'create', array_merge($tokenData, ['is_primary' => TRUE]));
-    $this->callAPISuccess('Phone', 'create', array_merge($tokenData, ['is_primary' => TRUE]));
-    $this->callAPISuccess('Email', 'create', array_merge($tokenData, ['is_primary' => TRUE]));
-    $this->callAPISuccess('Website', 'create', array_merge($tokenData, ['is_primary' => TRUE]));
-    $this->callAPISuccess('Im', 'create', ['is_primary' => TRUE, 'name' => $tokenData['im'], 'provider_id' => $tokenData['im_provider'], 'contact_id' => $tokenData['contact_id']]);
-    $this->callAPISuccess('OpenID', 'create', array_merge($tokenData, ['is_primary' => TRUE, 'contact_id' => $tokenData['contact_id'], 'openid' => $tokenData['openid']]));
+    $address = $this->setupContactFromTokeData($tokenData);
 
     CRM_Core_Smarty::singleton()->assign('pre_assigned_smarty', 'wee');
     // This string contains the 4 types of possible replaces just to be sure they
@@ -164,99 +159,49 @@ public function testContactTokens(): void {
       'subject' => $tokenString . ' ',
       'text' => $tokenString,
     ], FALSE, $tokenData['contact_id'], ['passed_smarty' => 'whoa']);
-    $checksum = substr($messageContent['html'], (strpos($messageContent['html'], 'cs=') + 3), 47);
-    $contact = Contact::get(FALSE)->addWhere('id', '=', $tokenData['contact_id'])->setSelect(['modified_date', 'employer_id'])->execute()->first();
     $expected = 'weewhoa
 Default Domain Name
-contact_type:Individual
-do_not_email:1
-do_not_phone:
-do_not_mail:1
-do_not_sms:1
-do_not_trade:1
-is_opt_out:1
-external_identifier:blah
-sort_name:Smith, Robert
-display_name:Mr. Robert Smith II
-nick_name:Bob
-image_URL:https://example.com
-preferred_communication_method:
-preferred_language:fr_CA
-preferred_mail_format:Both
-hash:xyz
-contact_source:Contact Source
-first_name:Robert
-middle_name:Frank
-last_name:Smith
-individual_prefix:Mr.
-individual_suffix:II
-formal_title:Dogsbody
-communication_style:Formal
-job_title:Busy person
-gender:Female
-birth_date:December 31st, 1998
-current_employer_id:' . $contact['employer_id'] . '
-contact_is_deleted:
-created_date:January 1st, 2020 12:00 AM
-modified_date:' . CRM_Utils_Date::customFormat($contact['modified_date']) . '
-addressee:Mr. Robert Frank Smith II
-email_greeting:Dear Robert
-postal_greeting:Dear Robert
-current_employer:Unit Test Organization
-location_type:Home
-address_id:' . $address['id'] . '
-street_address:Street Address
-street_number:123
-street_number_suffix:S
-street_name:Main St
-street_unit:45B
-supplemental_address_1:Round the corner
-supplemental_address_2:Up the road
-supplemental_address_3:By the big tree
-city:New York
-postal_code_suffix:4578
-postal_code:90210
-geo_code_1:48.858093
-geo_code_2:2.294694
-manual_geo_code:1
-address_name:The white house
-master_id:' . $tokenData['master_id'] . '
-county:
-state_province:TX
-country:United States
-phone:123-456
-phone_ext:77
-phone_type_id:
-phone_type:Mobile
-email:anthony_anderson@civicrm.org
-on_hold:
-signature_text:Yours sincerely
-signature_html:<p>Yours</p>
-im_provider:1
-im:IM Screen Name
-openid:OpenID
-world_region:America South, Central, North and Caribbean
-url:http://civicrm.org
-custom_1:Bobsled
-custom_2:Red
-custom_3:01/20/2021 12:00AM
-custom_4:999
-custom_5:<a href="http://civicrm.org" target="_blank">http://civicrm.org</a>
-custom_7:New Zealand
-custom_8:France, Canada
-custom_9:Mr. Spider Man II
-custom_10:Queensland
-custom_11:Victoria, New South Wales
-custom_12:Yes
-custom_13:Purple
-checksum:cs=' . $checksum . '
-contact_id:' . $tokenData['contact_id'] . '
 ';
+    $expected .= $this->getExpectedContactOutput($address['id'], $tokenData, $messageContent['html']);
     $this->assertEquals($expected, $messageContent['html']);
     $this->assertEquals($expected, $messageContent['text']);
     $this->assertEquals(rtrim(str_replace("\n", ' ', $expected)), $messageContent['subject']);
   }
 
+  /**
+   * Test the contact tokens rendered by the token processor.
+   *
+   * This test will be obsolete once the renderMessageTemplate
+   * function uses the token processor - at that point the test above
+   * will be testing the same thing.
+   *
+   * @throws \API_Exception
+   * @throws \CRM_Core_Exception
+   * @throws \CiviCRM_API3_Exception
+   */
+  public function testContactTokensRenderedByTokenProcessor(): void {
+    $this->createCustomGroupWithFieldsOfAllTypes([]);
+    $tokenData = $this->getAllContactTokens();
+    $address = $this->setupContactFromTokeData($tokenData);
+    $tokenString = '';
+    foreach (array_keys($tokenData) as $key) {
+      $tokenString .= "{$key}:{contact.{$key}}\n";
+    }
+    $tokenProcessor = new TokenProcessor(\Civi::dispatcher(), []);
+    $tokenProcessor->addMessage('html', $tokenString, 'text/html');
+    $tokenProcessor->addRow(['contactId' => $tokenData['contact_id']]);
+    $tokenProcessor->evaluate();
+    $rendered = '';
+    foreach ($tokenProcessor->getRows() as $row) {
+      $rendered = (string) $row->render('html');
+    }
+    $expected = $this->getExpectedContactOutput($address['id'], $tokenData, $rendered);
+    // @todo - this works better in token processor than in CRM_Core_Token.
+    // once synced we can fix $this->getExpectedContactOutput to return the right thing.
+    $expected = str_replace("preferred_communication_method:\n", "preferred_communication_method:Phone\n", $expected);
+    $this->assertEquals($expected, $rendered);
+  }
+
   /**
    * Gets the values needed to render domain tokens.
    *
@@ -402,4 +347,130 @@ public function getAllContactTokens(): array {
     ];
   }
 
+  /**
+   * @param array $tokenData
+   *
+   * @return array|int
+   * @throws \CRM_Core_Exception
+   */
+  protected function setupContactFromTokeData(array $tokenData) {
+    $this->callAPISuccess('Contact', 'create', $tokenData);
+    $address = $this->callAPISuccess('Address', 'create', array_merge($tokenData, ['is_primary' => TRUE]));
+    $this->callAPISuccess('Phone', 'create', array_merge($tokenData, ['is_primary' => TRUE]));
+    $this->callAPISuccess('Email', 'create', array_merge($tokenData, ['is_primary' => TRUE]));
+    $this->callAPISuccess('Website', 'create', array_merge($tokenData, ['is_primary' => TRUE]));
+    $this->callAPISuccess('Im', 'create', [
+      'is_primary' => TRUE,
+      'name' => $tokenData['im'],
+      'provider_id' => $tokenData['im_provider'],
+      'contact_id' => $tokenData['contact_id'],
+    ]);
+    $this->callAPISuccess('OpenID', 'create', array_merge($tokenData, [
+      'is_primary' => TRUE,
+      'contact_id' => $tokenData['contact_id'],
+      'openid' => $tokenData['openid'],
+    ]));
+    return $address;
+  }
+
+  /**
+   * Get the expected rendered string.
+   *
+   * @param int $id
+   * @param array $tokenData
+   * @param string $actualOutput
+   *
+   * @return string
+   * @throws \API_Exception
+   */
+  protected function getExpectedContactOutput($id, array $tokenData, string $actualOutput): string {
+    $checksum = substr($actualOutput, (strpos($actualOutput, 'cs=') + 3), 47);
+    $contact = Contact::get(FALSE)->addWhere('id', '=', $tokenData['contact_id'])->setSelect(['modified_date', 'employer_id'])->execute()->first();
+    $expected = 'contact_type:Individual
+do_not_email:1
+do_not_phone:
+do_not_mail:1
+do_not_sms:1
+do_not_trade:1
+is_opt_out:1
+external_identifier:blah
+sort_name:Smith, Robert
+display_name:Mr. Robert Smith II
+nick_name:Bob
+image_URL:https://example.com
+preferred_communication_method:
+preferred_language:fr_CA
+preferred_mail_format:Both
+hash:xyz
+contact_source:Contact Source
+first_name:Robert
+middle_name:Frank
+last_name:Smith
+individual_prefix:Mr.
+individual_suffix:II
+formal_title:Dogsbody
+communication_style:Formal
+job_title:Busy person
+gender:Female
+birth_date:December 31st, 1998
+current_employer_id:' . $contact['employer_id'] . '
+contact_is_deleted:
+created_date:January 1st, 2020 12:00 AM
+modified_date:' . CRM_Utils_Date::customFormat($contact['modified_date']) . '
+addressee:Mr. Robert Frank Smith II
+email_greeting:Dear Robert
+postal_greeting:Dear Robert
+current_employer:Unit Test Organization
+location_type:Home
+address_id:' . $id . '
+street_address:Street Address
+street_number:123
+street_number_suffix:S
+street_name:Main St
+street_unit:45B
+supplemental_address_1:Round the corner
+supplemental_address_2:Up the road
+supplemental_address_3:By the big tree
+city:New York
+postal_code_suffix:4578
+postal_code:90210
+geo_code_1:48.858093
+geo_code_2:2.294694
+manual_geo_code:1
+address_name:The white house
+master_id:' . $tokenData['master_id'] . '
+county:
+state_province:TX
+country:United States
+phone:123-456
+phone_ext:77
+phone_type_id:
+phone_type:Mobile
+email:anthony_anderson@civicrm.org
+on_hold:
+signature_text:Yours sincerely
+signature_html:&lt;p&gt;Yours&lt;/p&gt;
+im_provider:1
+im:IM Screen Name
+openid:OpenID
+world_region:America South, Central, North and Caribbean
+url:http://civicrm.org
+custom_1:Bobsled
+custom_2:Red
+custom_3:01/20/2021 12:00AM
+custom_4:999
+custom_5:<a href="http://civicrm.org" target="_blank">http://civicrm.org</a>
+custom_7:New Zealand
+custom_8:France, Canada
+custom_9:Mr. Spider Man II
+custom_10:Queensland
+custom_11:Victoria, New South Wales
+custom_12:Yes
+custom_13:Purple
+checksum:cs=' . $checksum . '
+contact_id:' . $tokenData['contact_id'] . '
+';
+    return $expected;
+  }
+
 }