From d8a1b674eb5365ca5931f7deaa0bd23f8f79bbcc Mon Sep 17 00:00:00 2001 From: eileen Date: Fri, 4 Dec 2020 21:09:27 +1300 Subject: [PATCH 01/63] Fix Payment edit form to use Payment.cancel & payment.create api --- CRM/Financial/Form/PaymentEdit.php | 35 +++++++------------ api/v3/Payment.php | 2 ++ .../CRM/Financial/Form/PaymentEditTest.php | 12 +++++-- 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/CRM/Financial/Form/PaymentEdit.php b/CRM/Financial/Form/PaymentEdit.php index ff85efa7351c..0e5b1261080d 100644 --- a/CRM/Financial/Form/PaymentEdit.php +++ b/CRM/Financial/Form/PaymentEdit.php @@ -174,6 +174,7 @@ public function postProcess() { * * @param array $submittedValues * + * @throws \CiviCRM_API3_Exception */ protected function submit($submittedValues) { // if payment instrument is changed then @@ -181,29 +182,17 @@ protected function submit($submittedValues) { // 2. Record a new financial transaction with new payment instrument // 3. Add EntityFinancialTrxn records to relate with corresponding financial item and contribution if ($submittedValues['payment_instrument_id'] != $this->_values['payment_instrument_id']) { - $previousFinanciaTrxn = $this->_values; - $newFinancialTrxn = $submittedValues; - unset($previousFinanciaTrxn['id'], $newFinancialTrxn['id']); - $previousFinanciaTrxn['trxn_date'] = CRM_Utils_Array::value('trxn_date', $submittedValues, date('YmdHis')); - $previousFinanciaTrxn['total_amount'] = -$previousFinanciaTrxn['total_amount']; - $previousFinanciaTrxn['net_amount'] = -$previousFinanciaTrxn['net_amount']; - $previousFinanciaTrxn['fee_amount'] = -$previousFinanciaTrxn['fee_amount']; - $previousFinanciaTrxn['contribution_id'] = $newFinancialTrxn['contribution_id'] = $this->_contributionID; + civicrm_api3('Payment', 'cancel', [ + 'id' => $this->_values['id'], + 'trxn_date' => $submittedValues['trxn_date'], + ]); + $newFinancialTrxn = $submittedValues; + unset($newFinancialTrxn['id']); $newFinancialTrxn['to_financial_account_id'] = CRM_Financial_BAO_FinancialTypeAccount::getInstrumentFinancialAccount($submittedValues['payment_instrument_id']); - foreach (['total_amount', 'currency', 'is_payment', 'status_id'] as $fieldName) { - $newFinancialTrxn[$fieldName] = $this->_values[$fieldName]; - } - - foreach ([$previousFinanciaTrxn, $newFinancialTrxn] as $financialTrxnParams) { - $financialTrxn = civicrm_api3('FinancialTrxn', 'create', $financialTrxnParams); - $trxnParams = [ - 'total_amount' => $financialTrxnParams['total_amount'], - 'contribution_id' => $this->_contributionID, - ]; - $contributionTotalAmount = CRM_Core_DAO::getFieldValue('CRM_Contribute_BAO_Contribution', $this->_contributionID, 'total_amount'); - CRM_Contribute_BAO_Contribution::assignProportionalLineItems($trxnParams, $financialTrxn['id'], $contributionTotalAmount); - } + $newFinancialTrxn['total_amount'] = $this->_values['total_amount']; + $newFinancialTrxn['currency'] = $this->_values['currency']; + civicrm_api3('Payment', 'create', $newFinancialTrxn); } else { // simply update the financial trxn @@ -217,8 +206,10 @@ protected function submit($submittedValues) { * Wrapper for unit testing the post process submit function. * * @param array $params + * + * @throws \CiviCRM_API3_Exception */ - public function testSubmit($params) { + public function testSubmit(array $params): void { $this->_id = $params['id']; $this->_contributionID = $params['contribution_id']; $this->_values = civicrm_api3('FinancialTrxn', 'getsingle', ['id' => $params['id']]); diff --git a/api/v3/Payment.php b/api/v3/Payment.php index bf10d26f1546..555c791c2dfd 100644 --- a/api/v3/Payment.php +++ b/api/v3/Payment.php @@ -94,6 +94,7 @@ function civicrm_api3_payment_cancel($params) { $eftParams = [ 'entity_table' => 'civicrm_contribution', 'financial_trxn_id' => $params['id'], + 'return' => ['entity', 'amount', 'entity_id', 'financial_trxn_id.check_number'], ]; $entity = civicrm_api3('EntityFinancialTrxn', 'getsingle', $eftParams); @@ -102,6 +103,7 @@ function civicrm_api3_payment_cancel($params) { 'contribution_id' => $entity['entity_id'], 'trxn_date' => $params['trxn_date'] ?? 'now', 'cancelled_payment_id' => $params['id'], + 'check_number' => $entity['financial_trxn_id.check_number'] ?? NULL, ]; foreach (['trxn_id', 'payment_instrument_id'] as $permittedParam) { diff --git a/tests/phpunit/CRM/Financial/Form/PaymentEditTest.php b/tests/phpunit/CRM/Financial/Form/PaymentEditTest.php index c415642e8b6e..c300732d54fb 100644 --- a/tests/phpunit/CRM/Financial/Form/PaymentEditTest.php +++ b/tests/phpunit/CRM/Financial/Form/PaymentEditTest.php @@ -28,6 +28,8 @@ public function setUp() { /** * Clean up after each test. + * + * @throws \CRM_Core_Exception */ public function tearDown() { $this->quickCleanUpFinancialEntities(); @@ -36,8 +38,12 @@ public function tearDown() { /** * Test the submit function of payment edit form. + * + * @throws \CRM_Core_Exception + * @throws \CiviCRM_API3_Exception + * @throws \Civi\Payment\Exception\PaymentProcessorException */ - public function testSubmitOnPaymentInstrumentChange() { + public function testSubmitOnPaymentInstrumentChange(): void { // First create a contribution using 'Check' as payment instrument $form = new CRM_Contribute_Form_Contribution(); $form->testSubmit([ @@ -61,7 +67,7 @@ public function testSubmitOnPaymentInstrumentChange() { 'payment_instrument_id' => CRM_Core_PseudoConstant::getKey('CRM_Contribute_BAO_Contribution', 'payment_instrument_id', 'Credit Card'), 'card_type_id' => CRM_Core_PseudoConstant::getKey('CRM_Financial_DAO_FinancialTrxn', 'card_type_id', 'Visa'), 'pan_truncation' => 1111, - 'trnx_id' => 'txn_12AAAA', + 'trxn_id' => 'txn_12AAAA', 'trxn_date' => date('Y-m-d H:i:s'), 'contribution_id' => $contribution['id'], ]; @@ -81,7 +87,7 @@ public function testSubmitOnPaymentInstrumentChange() { 'total_amount' => -50.00, 'financial_type' => 'Donation', 'payment_instrument' => 'Check', - 'status' => 'Completed', + 'status' => 'Refunded Label**', 'receive_date' => $params['trxn_date'], 'check_number' => '123XA', ], From 4f4d52d5007dea3552ef29535427192a0eac35cf Mon Sep 17 00:00:00 2001 From: eileen Date: Fri, 11 Dec 2020 10:08:55 +1300 Subject: [PATCH 02/63] dev/financial#162 Simplify decision as to whether to use a pdf on membership emails Currently the code that determines whether to attach a pdf for membership emails only attaches it as a pdf if the tax amount is non-zero This seems both confusing and pointless - the proposal here is to send as a pdf as long as is_email_pdf is true and invoicing is enabled I feel like 1 and 2 could be separated but they are currently linked so I would suggest any proposals to change that are a follow up --- CRM/Member/Form/Membership.php | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/CRM/Member/Form/Membership.php b/CRM/Member/Form/Membership.php index b5846d217778..6d40a7ac1e0c 100644 --- a/CRM/Member/Form/Membership.php +++ b/CRM/Member/Form/Membership.php @@ -986,20 +986,8 @@ public static function emailReceipt(&$form, &$formValues, &$membership, $customV $form->_receiptContactId = $formValues['contact_id']; } } - // @todo determine isEmailPdf in calling function. - $template = CRM_Core_Smarty::singleton(); - $taxAmt = $template->get_template_vars('dataArray'); - $eventTaxAmt = $template->get_template_vars('totalTaxAmount'); - $prefixValue = Civi::settings()->get('contribution_invoice_settings'); - $invoicing = $prefixValue['invoicing'] ?? NULL; - if ((!empty($taxAmt) || isset($eventTaxAmt)) && (isset($invoicing) && isset($prefixValue['is_email_pdf']))) { - $isEmailPdf = TRUE; - } - else { - $isEmailPdf = FALSE; - } - list($mailSend, $subject, $message, $html) = CRM_Core_BAO_MessageTemplate::sendTemplate( + CRM_Core_BAO_MessageTemplate::sendTemplate( [ 'groupName' => 'msg_tpl_workflow_membership', 'valueName' => 'membership_offline_receipt', @@ -1008,7 +996,7 @@ public static function emailReceipt(&$form, &$formValues, &$membership, $customV 'toName' => $form->_contributorDisplayName, 'toEmail' => $form->_contributorEmail, 'PDFFilename' => ts('receipt') . '.pdf', - 'isEmailPdf' => $isEmailPdf, + 'isEmailPdf' => Civi::settings()->get('invoicing') && Civi::settings()->get('is_email_pdf'), 'contributionId' => $formValues['contribution_id'], 'isTest' => (bool) ($form->_action & CRM_Core_Action::PREVIEW), ] From 6fd75357518454c744701bccd5d28e96cfcc084e Mon Sep 17 00:00:00 2001 From: eileen Date: Mon, 14 Dec 2020 14:26:16 +1300 Subject: [PATCH 03/63] Fix Contribution.create to not attempt to set contacts on activity update In udpate cases the original allocations were more correct --- CRM/Contribute/BAO/Contribution.php | 14 ++++++++++---- tests/phpunit/CRM/Activity/Form/SearchTest.php | 7 +++++-- tests/phpunit/api/v3/JobTest.php | 3 ++- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/CRM/Contribute/BAO/Contribution.php b/CRM/Contribute/BAO/Contribution.php index 7f825adfa016..9c58190677b3 100644 --- a/CRM/Contribute/BAO/Contribution.php +++ b/CRM/Contribute/BAO/Contribution.php @@ -517,18 +517,24 @@ public static function create(&$params) { ])->execute()->first(); $campaignParams = isset($params['campaign_id']) ? ['campaign_id' => ($params['campaign_id'] ?? NULL)] : []; - Activity::save(FALSE)->addRecord(array_merge([ + $activityParams = array_merge([ 'activity_type_id:name' => 'Contribution', 'source_record_id' => $contribution->id, - 'source_contact_id' => CRM_Core_Session::getLoggedInContactID() ?: $contribution->contact_id, - 'target_contact_id' => CRM_Core_Session::getLoggedInContactID() ? [$contribution->contact_id] : [], 'activity_date_time' => $contribution->receive_date, 'is_test' => (bool) $contribution->is_test, 'status_id:name' => $isCompleted ? 'Completed' : 'Scheduled', 'skipRecentView' => TRUE, 'subject' => CRM_Activity_BAO_Activity::getActivitySubject($contribution), 'id' => $existingActivity['id'] ?? NULL, - ], $campaignParams))->execute(); + ], $campaignParams); + if (!$activityParams['id']) { + // Don't set target contacts on update as these will have been + // correctly created and we risk overwriting them with + // 'best guess' params. + $activityParams['source_contact_id'] = (int) ($params['source_contact_id'] ?? (CRM_Core_Session::getLoggedInContactID() ?: $contribution->contact_id)); + $activityParams['target_contact_id'] = ($activityParams['source_contact_id'] === (int) $contribution->contact_id) ? [] : [$contribution->contact_id]; + } + Activity::save(FALSE)->addRecord($activityParams)->execute(); } // do not add to recent items for import, CRM-4399 diff --git a/tests/phpunit/CRM/Activity/Form/SearchTest.php b/tests/phpunit/CRM/Activity/Form/SearchTest.php index de46b7eeebf1..2c7d4faa3224 100644 --- a/tests/phpunit/CRM/Activity/Form/SearchTest.php +++ b/tests/phpunit/CRM/Activity/Form/SearchTest.php @@ -24,9 +24,12 @@ public function tearDown() { } /** - * Test submitted the search form. + * Test submitted the search form. + * + * @throws \CRM_Core_Exception + * @throws \CiviCRM_API3_Exception */ - public function testSearch() { + public function testSearch(): void { $form = new CRM_Activity_Form_Search(); $_SERVER['REQUEST_METHOD'] = 'GET'; diff --git a/tests/phpunit/api/v3/JobTest.php b/tests/phpunit/api/v3/JobTest.php index 5bc0ccb8fddb..46e2a09ca212 100644 --- a/tests/phpunit/api/v3/JobTest.php +++ b/tests/phpunit/api/v3/JobTest.php @@ -462,8 +462,9 @@ public function testBatchMergeWorks($dataSet) { * Note the group combinations & expected results: * * @throws \CRM_Core_Exception + * @throws \CiviCRM_API3_Exception */ - public function testBatchMergeWithAssets() { + public function testBatchMergeWithAssets(): void { $contactID = $this->individualCreate(); $contact2ID = $this->individualCreate(); $this->contributionCreate(['contact_id' => $contactID]); From 148d475e25d0dd9a98f140dd29fb38a26a93efc7 Mon Sep 17 00:00:00 2001 From: Matthew Wire Date: Mon, 14 Dec 2020 11:13:54 +0000 Subject: [PATCH 04/63] Add an eventID to the pre/post Insert/Update events so they can be matched together --- CRM/Core/DAO.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CRM/Core/DAO.php b/CRM/Core/DAO.php index 8b55524be8b9..70d248bfb010 100644 --- a/CRM/Core/DAO.php +++ b/CRM/Core/DAO.php @@ -615,9 +615,11 @@ public function table() { * @return CRM_Core_DAO */ public function save($hook = TRUE) { + $eventID = uniqid(); if (!empty($this->id)) { if ($hook) { $preEvent = new \Civi\Core\DAO\Event\PreUpdate($this); + $preEvent->eventID = $eventID; \Civi::dispatcher()->dispatch("civi.dao.preUpdate", $preEvent); } @@ -625,6 +627,7 @@ public function save($hook = TRUE) { if ($hook) { $event = new \Civi\Core\DAO\Event\PostUpdate($this, $result); + $event->eventID = $eventID; \Civi::dispatcher()->dispatch("civi.dao.postUpdate", $event); } $this->clearDbColumnValueCache(); @@ -632,6 +635,7 @@ public function save($hook = TRUE) { else { if ($hook) { $preEvent = new \Civi\Core\DAO\Event\PreUpdate($this); + $preEvent->eventID = $eventID; \Civi::dispatcher()->dispatch("civi.dao.preInsert", $preEvent); } @@ -639,6 +643,7 @@ public function save($hook = TRUE) { if ($hook) { $event = new \Civi\Core\DAO\Event\PostUpdate($this, $result); + $event->eventID = $eventID; \Civi::dispatcher()->dispatch("civi.dao.postInsert", $event); } } From d581861e481bb97c24f9d102e0428c9894d6e408 Mon Sep 17 00:00:00 2001 From: eileen Date: Tue, 15 Dec 2020 08:40:50 +1300 Subject: [PATCH 05/63] Remove functions from EmailCommon that were moved to the trait --- CRM/Contact/Form/Task/EmailCommon.php | 521 -------------------------- 1 file changed, 521 deletions(-) diff --git a/CRM/Contact/Form/Task/EmailCommon.php b/CRM/Contact/Form/Task/EmailCommon.php index 1244b44f3a0d..02a4df9d5459 100644 --- a/CRM/Contact/Form/Task/EmailCommon.php +++ b/CRM/Contact/Form/Task/EmailCommon.php @@ -22,14 +22,6 @@ */ class CRM_Contact_Form_Task_EmailCommon { - const MAX_EMAILS_KILL_SWITCH = 50; - - public $_contactDetails = []; - - public $_allContactDetails = []; - - public $_toContactEmails = []; - /** * Pre Process Form Addresses to be used in Quickform * @@ -79,243 +71,6 @@ public static function preProcessFromAddress(&$form, $bounce = TRUE) { $form->setDefaults($defaults); } - /** - * Build the form object. - * - * @param CRM_Core_Form $form - * - * @throws \CRM_Core_Exception - */ - public static function buildQuickForm(&$form) { - CRM_Core_Error::deprecatedFunctionWarning('This code is no longer used in core and will be removed'); - $toArray = $ccArray = $bccArray = []; - $suppressedEmails = 0; - //here we are getting logged in user id as array but we need target contact id. CRM-5988 - $cid = $form->get('cid'); - if ($cid) { - $form->_contactIds = explode(',', $cid); - } - if (count($form->_contactIds) > 1) { - $form->_single = FALSE; - } - CRM_Contact_Form_Task_EmailCommon::bounceIfSimpleMailLimitExceeded(count($form->_contactIds)); - - $emailAttributes = [ - 'class' => 'huge', - ]; - $to = $form->add('text', 'to', ts('To'), $emailAttributes, TRUE); - $cc = $form->add('text', 'cc_id', ts('CC'), $emailAttributes); - $bcc = $form->add('text', 'bcc_id', ts('BCC'), $emailAttributes); - - if ($to->getValue()) { - $form->_toContactIds = $form->_contactIds = []; - } - $setDefaults = TRUE; - if (property_exists($form, '_context') && $form->_context == 'standalone') { - $setDefaults = FALSE; - } - - $elements = ['to', 'cc', 'bcc']; - $form->_allContactIds = $form->_toContactIds = $form->_contactIds; - foreach ($elements as $element) { - if ($$element->getValue()) { - - foreach (self::getEmails($$element) as $value) { - $contactId = $value['contact_id']; - $email = $value['email']; - if ($contactId) { - switch ($element) { - case 'to': - $form->_contactIds[] = $form->_toContactIds[] = $contactId; - $form->_toContactEmails[] = $email; - break; - - case 'cc': - $form->_ccContactIds[] = $contactId; - break; - - case 'bcc': - $form->_bccContactIds[] = $contactId; - break; - } - - $form->_allContactIds[] = $contactId; - } - } - - $setDefaults = TRUE; - } - } - - //get the group of contacts as per selected by user in case of Find Activities - if (!empty($form->_activityHolderIds)) { - $contact = $form->get('contacts'); - $form->_allContactIds = $form->_contactIds = $contact; - } - - // check if we need to setdefaults and check for valid contact emails / communication preferences - if (is_array($form->_allContactIds) && $setDefaults) { - $returnProperties = [ - 'sort_name' => 1, - 'email' => 1, - 'do_not_email' => 1, - 'is_deceased' => 1, - 'on_hold' => 1, - 'display_name' => 1, - 'preferred_mail_format' => 1, - ]; - - // get the details for all selected contacts ( to, cc and bcc contacts ) - list($form->_contactDetails) = CRM_Utils_Token::getTokenDetails($form->_allContactIds, - $returnProperties, - FALSE, - FALSE - ); - - // make a copy of all contact details - $form->_allContactDetails = $form->_contactDetails; - - // perform all validations on unique contact Ids - foreach (array_unique($form->_allContactIds) as $key => $contactId) { - $value = $form->_contactDetails[$contactId]; - if ($value['do_not_email'] || empty($value['email']) || !empty($value['is_deceased']) || $value['on_hold']) { - $suppressedEmails++; - - // unset contact details for contacts that we won't be sending email. This is prevent extra computation - // during token evaluation etc. - unset($form->_contactDetails[$contactId]); - } - else { - $email = $value['email']; - - // build array's which are used to setdefaults - if (in_array($contactId, $form->_toContactIds)) { - $form->_toContactDetails[$contactId] = $form->_contactDetails[$contactId]; - // If a particular address has been specified as the default, use that instead of contact's primary email - if (!empty($form->_toEmail) && $form->_toEmail['contact_id'] == $contactId) { - $email = $form->_toEmail['email']; - } - $toArray[] = [ - 'text' => '"' . $value['sort_name'] . '" <' . $email . '>', - 'id' => "$contactId::{$email}", - ]; - } - elseif (in_array($contactId, $form->_ccContactIds)) { - $ccArray[] = [ - 'text' => '"' . $value['sort_name'] . '" <' . $email . '>', - 'id' => "$contactId::{$email}", - ]; - } - elseif (in_array($contactId, $form->_bccContactIds)) { - $bccArray[] = [ - 'text' => '"' . $value['sort_name'] . '" <' . $email . '>', - 'id' => "$contactId::{$email}", - ]; - } - } - } - - if (empty($toArray)) { - CRM_Core_Error::statusBounce(ts('Selected contact(s) do not have a valid email address, or communication preferences specify DO NOT EMAIL, or they are deceased or Primary email address is On Hold.')); - } - } - - $form->assign('toContact', json_encode($toArray)); - $form->assign('ccContact', json_encode($ccArray)); - $form->assign('bccContact', json_encode($bccArray)); - - $form->assign('suppressedEmails', $suppressedEmails); - - $form->assign('totalSelectedContacts', count($form->_contactIds)); - - $form->add('text', 'subject', ts('Subject'), 'size=50 maxlength=254', TRUE); - - $form->add('select', 'from_email_address', ts('From'), $form->_fromEmails, TRUE); - - CRM_Mailing_BAO_Mailing::commonCompose($form); - - // add attachments - CRM_Core_BAO_File::buildAttachment($form, NULL); - - if ($form->_single) { - // also fix the user context stack - if ($form->_caseId) { - $ccid = CRM_Core_DAO::getFieldValue('CRM_Case_DAO_CaseContact', $form->_caseId, - 'contact_id', 'case_id' - ); - $url = CRM_Utils_System::url('civicrm/contact/view/case', - "&reset=1&action=view&cid={$ccid}&id={$form->_caseId}" - ); - } - elseif ($form->_context) { - $url = CRM_Utils_System::url('civicrm/dashboard', 'reset=1'); - } - else { - $url = CRM_Utils_System::url('civicrm/contact/view', - "&show=1&action=browse&cid={$form->_contactIds[0]}&selectedChild=activity" - ); - } - - $session = CRM_Core_Session::singleton(); - $session->replaceUserContext($url); - $form->addDefaultButtons(ts('Send Email'), 'upload', 'cancel'); - } - else { - $form->addDefaultButtons(ts('Send Email'), 'upload'); - } - - $fields = [ - 'followup_assignee_contact_id' => [ - 'type' => 'entityRef', - 'label' => ts('Assigned to'), - 'attributes' => [ - 'multiple' => TRUE, - 'create' => TRUE, - 'api' => ['params' => ['is_deceased' => 0]], - ], - ], - 'followup_activity_type_id' => [ - 'type' => 'select', - 'label' => ts('Followup Activity'), - 'attributes' => ['' => '- ' . ts('select activity') . ' -'] + CRM_Core_PseudoConstant::ActivityType(FALSE), - 'extra' => ['class' => 'crm-select2'], - ], - 'followup_activity_subject' => [ - 'type' => 'text', - 'label' => ts('Subject'), - 'attributes' => CRM_Core_DAO::getAttribute('CRM_Activity_DAO_Activity', - 'subject' - ), - ], - ]; - - //add followup date - $form->add('datepicker', 'followup_date', ts('in')); - - foreach ($fields as $field => $values) { - if (!empty($fields[$field])) { - $attribute = $values['attributes'] ?? NULL; - $required = !empty($values['required']); - - if ($values['type'] == 'select' && empty($attribute)) { - $form->addSelect($field, ['entity' => 'activity'], $required); - } - elseif ($values['type'] == 'entityRef') { - $form->addEntityRef($field, $values['label'], $attribute, $required); - } - else { - $form->add($values['type'], $field, $values['label'], $attribute, $required, CRM_Utils_Array::value('extra', $values)); - } - } - } - - //Added for CRM-15984: Add campaign field - CRM_Campaign_BAO_Campaign::addCampaign($form); - - $form->addFormRule(['CRM_Contact_Form_Task_EmailCommon', 'formRule'], $form); - CRM_Core_Resources::singleton()->addScriptFile('civicrm', 'templates/CRM/Contact/Form/Task/EmailCommon.js', 0, 'html-header'); - } - /** * Form rule. * @@ -346,280 +101,4 @@ public static function formRule($fields, $dontCare, $self) { return empty($errors) ? TRUE : $errors; } - /** - * Process the form after the input has been submitted and validated. - * - * @param CRM_Core_Form $form - * - * @throws \CRM_Core_Exception - * @throws \CiviCRM_API3_Exception - * @throws \Civi\API\Exception\UnauthorizedException - */ - public static function postProcess(&$form) { - CRM_Core_Error::deprecatedFunctionWarning('This code is no longer used in core and will be removed'); - - self::bounceIfSimpleMailLimitExceeded(count($form->_contactIds)); - - // check and ensure that - $formValues = $form->controller->exportValues($form->getName()); - self::submit($form, $formValues); - } - - /** - * Submit the form values. - * - * This is also accessible for testing. - * - * @param CRM_Core_Form $form - * @param array $formValues - * - * @throws \CRM_Core_Exception - * @throws \CiviCRM_API3_Exception - * @throws \Civi\API\Exception\UnauthorizedException - */ - public static function submit(&$form, $formValues) { - CRM_Core_Error::deprecatedFunctionWarning('This code is no longer used in core and will be removed'); - - self::saveMessageTemplate($formValues); - - $from = $formValues['from_email_address'] ?? NULL; - // dev/core#357 User Emails are keyed by their id so that the Signature is able to be added - // If we have had a contact email used here the value returned from the line above will be the - // numerical key where as $from for use in the sendEmail in Activity needs to be of format of "To Name" - $from = CRM_Utils_Mail::formatFromAddress($from); - $subject = $formValues['subject']; - - // CRM-13378: Append CC and BCC information at the end of Activity Details and format cc and bcc fields - $elements = ['cc_id', 'bcc_id']; - $additionalDetails = NULL; - $ccValues = $bccValues = []; - foreach ($elements as $element) { - if (!empty($formValues[$element])) { - $allEmails = explode(',', $formValues[$element]); - foreach ($allEmails as $value) { - list($contactId, $email) = explode('::', $value); - $contactURL = CRM_Utils_System::url('civicrm/contact/view', "reset=1&force=1&cid={$contactId}", TRUE); - switch ($element) { - case 'cc_id': - $ccValues['email'][] = '"' . $form->_contactDetails[$contactId]['sort_name'] . '" <' . $email . '>'; - $ccValues['details'][] = "" . $form->_contactDetails[$contactId]['display_name'] . ""; - break; - - case 'bcc_id': - $bccValues['email'][] = '"' . $form->_contactDetails[$contactId]['sort_name'] . '" <' . $email . '>'; - $bccValues['details'][] = "" . $form->_contactDetails[$contactId]['display_name'] . ""; - break; - } - } - } - } - - $cc = $bcc = ''; - if (!empty($ccValues)) { - $cc = implode(',', $ccValues['email']); - $additionalDetails .= "\ncc : " . implode(", ", $ccValues['details']); - } - if (!empty($bccValues)) { - $bcc = implode(',', $bccValues['email']); - $additionalDetails .= "\nbcc : " . implode(", ", $bccValues['details']); - } - - // CRM-5916: prepend case id hash to CiviCase-originating emails’ subjects - if (isset($form->_caseId) && is_numeric($form->_caseId)) { - $hash = substr(sha1(CIVICRM_SITE_KEY . $form->_caseId), 0, 7); - $subject = "[case #$hash] $subject"; - } - - $attachments = []; - CRM_Core_BAO_File::formatAttachment($formValues, - $attachments, - NULL, NULL - ); - - // format contact details array to handle multiple emails from same contact - $formattedContactDetails = []; - $tempEmails = []; - foreach ($form->_contactIds as $key => $contactId) { - // if we dont have details on this contactID, we should ignore - // potentially this is due to the contact not wanting to receive email - if (!isset($form->_contactDetails[$contactId])) { - continue; - } - $email = $form->_toContactEmails[$key]; - // prevent duplicate emails if same email address is selected CRM-4067 - // we should allow same emails for different contacts - $emailKey = "{$contactId}::{$email}"; - if (!in_array($emailKey, $tempEmails)) { - $tempEmails[] = $emailKey; - $details = $form->_contactDetails[$contactId]; - $details['email'] = $email; - unset($details['email_id']); - $formattedContactDetails[] = $details; - } - } - - $contributionIds = []; - if ($form->getVar('_contributionIds')) { - $contributionIds = $form->getVar('_contributionIds'); - } - - // send the mail - list($sent, $activityId) = CRM_Activity_BAO_Activity::sendEmail( - $formattedContactDetails, - $subject, - $formValues['text_message'], - $formValues['html_message'], - NULL, - NULL, - $from, - $attachments, - $cc, - $bcc, - array_keys($form->_toContactDetails), - $additionalDetails, - $contributionIds, - CRM_Utils_Array::value('campaign_id', $formValues), - $form->getVar('_caseId') - ); - - $followupStatus = ''; - if ($sent) { - $followupActivity = NULL; - if (!empty($formValues['followup_activity_type_id'])) { - $params['followup_activity_type_id'] = $formValues['followup_activity_type_id']; - $params['followup_activity_subject'] = $formValues['followup_activity_subject']; - $params['followup_date'] = $formValues['followup_date']; - $params['target_contact_id'] = $form->_contactIds; - $params['followup_assignee_contact_id'] = explode(',', $formValues['followup_assignee_contact_id']); - $followupActivity = CRM_Activity_BAO_Activity::createFollowupActivity($activityId, $params); - $followupStatus = ts('A followup activity has been scheduled.'); - - if (Civi::settings()->get('activity_assignee_notification')) { - if ($followupActivity) { - $mailToFollowupContacts = []; - $assignee = [$followupActivity->id]; - $assigneeContacts = CRM_Activity_BAO_ActivityAssignment::getAssigneeNames($assignee, TRUE, FALSE); - foreach ($assigneeContacts as $values) { - $mailToFollowupContacts[$values['email']] = $values; - } - - $sentFollowup = CRM_Activity_BAO_Activity::sendToAssignee($followupActivity, $mailToFollowupContacts); - if ($sentFollowup) { - $followupStatus .= '
' . ts("A copy of the follow-up activity has also been sent to follow-up assignee contacts(s)."); - } - } - } - } - - $count_success = count($form->_toContactDetails); - CRM_Core_Session::setStatus(ts('One message was sent successfully. ', [ - 'plural' => '%count messages were sent successfully. ', - 'count' => $count_success, - ]) . $followupStatus, ts('Message Sent', ['plural' => 'Messages Sent', 'count' => $count_success]), 'success'); - } - - // Display the name and number of contacts for those email is not sent. - // php 5.4 throws out a notice since the values of these below arrays are arrays. - // the behavior is not documented in the php manual, but it does the right thing - // suppressing the notices to get things in good shape going forward - $emailsNotSent = @array_diff_assoc($form->_allContactDetails, $form->_contactDetails); - - if ($emailsNotSent) { - $not_sent = []; - foreach ($emailsNotSent as $contactId => $values) { - $displayName = $values['display_name']; - $email = $values['email']; - $contactViewUrl = CRM_Utils_System::url('civicrm/contact/view', "reset=1&cid=$contactId"); - $not_sent[] = "$displayName" . ($values['on_hold'] ? '(' . ts('on hold') . ')' : ''); - } - $status = '(' . ts('because no email address on file or communication preferences specify DO NOT EMAIL or Contact is deceased or Primary email address is On Hold') . ')
  • ' . implode('
  • ', $not_sent) . '
'; - CRM_Core_Session::setStatus($status, ts('One Message Not Sent', [ - 'count' => count($emailsNotSent), - 'plural' => '%count Messages Not Sent', - ]), 'info'); - } - - if (isset($form->_caseId)) { - // if case-id is found in the url, create case activity record - $cases = explode(',', $form->_caseId); - foreach ($cases as $key => $val) { - if (is_numeric($val)) { - $caseParams = [ - 'activity_id' => $activityId, - 'case_id' => $val, - ]; - CRM_Case_BAO_Case::processCaseActivity($caseParams); - } - } - } - } - - /** - * Save the template if update selected. - * - * @param array $formValues - * - * @throws \CiviCRM_API3_Exception - * @throws \Civi\API\Exception\UnauthorizedException - */ - protected static function saveMessageTemplate($formValues) { - CRM_Core_Error::deprecatedFunctionWarning('This code is no longer used in core and will be removed'); - - if (!empty($formValues['saveTemplate']) || !empty($formValues['updateTemplate'])) { - $messageTemplate = [ - 'msg_text' => $formValues['text_message'], - 'msg_html' => $formValues['html_message'], - 'msg_subject' => $formValues['subject'], - 'is_active' => TRUE, - ]; - - if (!empty($formValues['saveTemplate'])) { - $messageTemplate['msg_title'] = $formValues['saveTemplateName']; - CRM_Core_BAO_MessageTemplate::add($messageTemplate); - } - - if (!empty($formValues['template']) && !empty($formValues['updateTemplate'])) { - $messageTemplate['id'] = $formValues['template']; - unset($messageTemplate['msg_title']); - CRM_Core_BAO_MessageTemplate::add($messageTemplate); - } - } - } - - /** - * Bounce if there are more emails than permitted. - * - * @param int $count - * The number of emails the user is attempting to send - */ - public static function bounceIfSimpleMailLimitExceeded($count) { - CRM_Core_Error::deprecatedFunctionWarning('This code is no longer used in core and will be removed'); - - $limit = Civi::settings()->get('simple_mail_limit'); - if ($count > $limit) { - CRM_Core_Error::statusBounce(ts('Please do not use this task to send a lot of emails (greater than %1). Many countries have legal requirements when sending bulk emails and the CiviMail framework has opt out functionality and domain tokens to help meet these.', - [1 => $limit] - )); - } - } - - /** - * Get the emails from the added element. - * - * @param HTML_QuickForm_Element $element - * - * @return array - */ - protected static function getEmails($element): array { - CRM_Core_Error::deprecatedFunctionWarning('This code is no longer used in core and will be removed'); - - $allEmails = explode(',', $element->getValue()); - $return = []; - foreach ($allEmails as $value) { - $values = explode('::', $value); - $return[] = ['contact_id' => $values[0], 'email' => $values[1]]; - } - return $return; - } - } From c2978fe0dc007daaf7bd43065eaa5e4df77fe152 Mon Sep 17 00:00:00 2001 From: eileen Date: Fri, 11 Dec 2020 15:35:46 +1300 Subject: [PATCH 06/63] dev/financial#163 Comment all the places where removal of contributionTypeID proposed --- CRM/Contribute/BAO/ContributionPage.php | 1 - CRM/Contribute/Form/Contribution.php | 1 - CRM/Contribute/Form/Contribution/Confirm.php | 3 +-- CRM/Event/Form/Registration/Confirm.php | 2 -- CRM/Member/Form/Membership.php | 1 - 5 files changed, 1 insertion(+), 7 deletions(-) diff --git a/CRM/Contribute/BAO/ContributionPage.php b/CRM/Contribute/BAO/ContributionPage.php index e5235acaea8f..90e8ec6ef594 100644 --- a/CRM/Contribute/BAO/ContributionPage.php +++ b/CRM/Contribute/BAO/ContributionPage.php @@ -369,7 +369,6 @@ public static function sendMail($contactID, $values, $isTest = FALSE, $returnMes $values['financial_type_id']); // Legacy support $tplParams['contributionTypeName'] = $tplParams['financialTypeName']; - $tplParams['contributionTypeId'] = $values['financial_type_id']; } if ($contributionPageId = CRM_Utils_Array::value('id', $values)) { diff --git a/CRM/Contribute/Form/Contribution.php b/CRM/Contribute/Form/Contribution.php index 1fc9dc406399..54c6be460011 100644 --- a/CRM/Contribute/Form/Contribution.php +++ b/CRM/Contribute/Form/Contribution.php @@ -1114,7 +1114,6 @@ protected function processCreditCard($submittedValues, $lineItem, $contactID) { ); $paymentParams['contributionID'] = $contribution->id; - $paymentParams['contributionTypeID'] = $contribution->financial_type_id; $paymentParams['contributionPageID'] = $contribution->contribution_page_id; $paymentParams['contributionRecurID'] = $contribution->contribution_recur_id; diff --git a/CRM/Contribute/Form/Contribution/Confirm.php b/CRM/Contribute/Form/Contribution/Confirm.php index fda165b2a03b..45a59aa9caef 100644 --- a/CRM/Contribute/Form/Contribution/Confirm.php +++ b/CRM/Contribute/Form/Contribution/Confirm.php @@ -2558,8 +2558,7 @@ public static function processConfirm( // add some financial type details to the params list // if folks need to use it - //CRM-15297 deprecate contributionTypeID - $paymentParams['financial_type_id'] = $paymentParams['financialTypeID'] = $paymentParams['contributionTypeID'] = $financialType->id; + $paymentParams['financial_type_id'] = $paymentParams['financialTypeID'] = $financialType->id; //CRM-15297 - contributionType is obsolete - pass financial type as well so people can deprecate it $paymentParams['financialType_name'] = $paymentParams['contributionType_name'] = $form->_params['contributionType_name'] = $financialType->name; //CRM-11456 diff --git a/CRM/Event/Form/Registration/Confirm.php b/CRM/Event/Form/Registration/Confirm.php index 14afb06fe5ba..d83cad562f70 100644 --- a/CRM/Event/Form/Registration/Confirm.php +++ b/CRM/Event/Form/Registration/Confirm.php @@ -563,11 +563,9 @@ public function postProcess() { //passing contribution id is already registered. $contribution = $this->processContribution($this, $value, $result, $contactID, $pending, $this->_paymentProcessor); $value['contributionID'] = $contribution->id; - $value['contributionTypeID'] = $contribution->financial_type_id; $value['receive_date'] = $contribution->receive_date; $value['trxn_id'] = $contribution->trxn_id; $value['contributionID'] = $contribution->id; - $value['contributionTypeID'] = $contribution->financial_type_id; } $value['contactID'] = $contactID; $value['eventID'] = $this->_eventId; diff --git a/CRM/Member/Form/Membership.php b/CRM/Member/Form/Membership.php index 3d6e1c8230f6..2439dec25c6d 100644 --- a/CRM/Member/Form/Membership.php +++ b/CRM/Member/Form/Membership.php @@ -1339,7 +1339,6 @@ public function submit() { $paymentParams['contactID'] = $this->_contactID; $paymentParams['contributionID'] = $contribution->id; - $paymentParams['contributionTypeID'] = $contribution->financial_type_id; $paymentParams['contributionPageID'] = $contribution->contribution_page_id; $paymentParams['contributionRecurID'] = $contribution->contribution_recur_id; $params['contribution_id'] = $paymentParams['contributionID']; From 170b7f0b9ca6b7a14da43ca7dffacc14e263d161 Mon Sep 17 00:00:00 2001 From: Coleman Watts Date: Tue, 15 Dec 2020 10:02:26 -0500 Subject: [PATCH 07/63] Greenwich: Fix Select2 free-tagging css bug --- ext/greenwich/scss/_tweaks.scss | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ext/greenwich/scss/_tweaks.scss b/ext/greenwich/scss/_tweaks.scss index 0bab3e6e4e3f..fe3a79d1cf7d 100644 --- a/ext/greenwich/scss/_tweaks.scss +++ b/ext/greenwich/scss/_tweaks.scss @@ -6,3 +6,7 @@ label input[type=checkbox]:not(:checked) + * { font-weight: normal; } +/* Fix tagging-style select2 */ +.select2-choices { + margin-bottom: 0; +} From be8966569ad623252e2bba56f65a93c894fc5118 Mon Sep 17 00:00:00 2001 From: "Laryn - CEDC.org" Date: Mon, 14 Dec 2020 15:36:10 -0600 Subject: [PATCH 08/63] dev/core#2211 Make sure addressee field fits column Truncate if addressee field goes beyond the available 255 characters in an export that merges contacts by shared address. --- CRM/Export/BAO/ExportProcessor.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CRM/Export/BAO/ExportProcessor.php b/CRM/Export/BAO/ExportProcessor.php index 11e26f65eda3..3ba681e45648 100644 --- a/CRM/Export/BAO/ExportProcessor.php +++ b/CRM/Export/BAO/ExportProcessor.php @@ -1985,7 +1985,7 @@ public function mergeSameAddress() { WHERE id = %4 "; $params = [ - 1 => [$values['addressee'], 'String'], + 1 => [CRM_Utils_String::ellipsify($values['addressee'], 255), 'String'], 2 => [$values['postalGreeting'], 'String'], 3 => [$values['emailGreeting'], 'String'], 4 => [$masterID, 'Integer'], From 5d24295f213c00c479bdf07435f7805ae8fb3d61 Mon Sep 17 00:00:00 2001 From: eileen Date: Wed, 16 Dec 2020 09:48:11 +1300 Subject: [PATCH 09/63] Remove legacy tpl assigns These really look like they were copied over from participant forms - they are not used in the membership tpl receipts --- CRM/Member/Form/Membership.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/CRM/Member/Form/Membership.php b/CRM/Member/Form/Membership.php index 3d6e1c8230f6..5840a4c37a1d 100644 --- a/CRM/Member/Form/Membership.php +++ b/CRM/Member/Form/Membership.php @@ -932,9 +932,6 @@ public static function emailReceipt(&$form, &$formValues, &$membership, $customV $valuesForForm = CRM_Contribute_Form_AbstractEditPayment::formatCreditCardDetails($form->_params); $form->assignVariables($valuesForForm, ['credit_card_exp_date', 'credit_card_type', 'credit_card_number']); - - $form->assign('contributeMode', 'direct'); - $form->assign('isAmountzero', 0); $form->assign('is_pay_later', 0); $form->assign('isPrimary', 1); } From 93ff2b21ecc826196cf9762a9b5ad514e065d00e Mon Sep 17 00:00:00 2001 From: Rich Lott / Artful Robot Date: Tue, 15 Dec 2020 21:06:00 +0000 Subject: [PATCH 10/63] Fix lab issue 2254: cannot set is_bulkmail from UI --- CRM/Core/BAO/Email.php | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/CRM/Core/BAO/Email.php b/CRM/Core/BAO/Email.php index aa0fdc956a88..b4ed167c0fc8 100644 --- a/CRM/Core/BAO/Email.php +++ b/CRM/Core/BAO/Email.php @@ -44,13 +44,17 @@ public static function create($params) { $email->email = $strtolower($email->email); } - /* - * since we're setting bulkmail for 1 of this contact's emails, first reset all their other emails to is_bulkmail false - * We shouldn't set the current email to false even though we - * are about to reset it to avoid contaminating the changelog if logging is enabled. - * (only 1 email address can have is_bulkmail = true) - */ - if ($email->is_bulkmail && !empty($params['contact_id']) && !self::isMultipleBulkMail()) { + // + // Since we're setting bulkmail for 1 of this contact's emails, first reset + // all their other emails to is_bulkmail false. We shouldn't set the current + // email to false even though we are about to reset it to avoid + // contaminating the changelog if logging is enabled. (only 1 email + // address can have is_bulkmail = true) + // + // Note setting a the is_bulkmail to '' in $params results in $email->is_bulkmail === 'null'. + // @see https://lab.civicrm.org/dev/core/-/issues/2254 + // + if ($email->is_bulkmail == 1 && !empty($params['contact_id']) && !self::isMultipleBulkMail()) { $sql = " UPDATE civicrm_email SET is_bulkmail = 0 From 82c4e0051778713d83ebcd634a51e03aeea87e1a Mon Sep 17 00:00:00 2001 From: eileen Date: Tue, 15 Dec 2020 15:25:56 +1300 Subject: [PATCH 11/63] Remove always true parameter --- CRM/Member/Form/Membership.php | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/CRM/Member/Form/Membership.php b/CRM/Member/Form/Membership.php index caa74005d93e..5c1e1e6a1df6 100644 --- a/CRM/Member/Form/Membership.php +++ b/CRM/Member/Form/Membership.php @@ -1325,8 +1325,7 @@ public function submit() { ], $financialType, FALSE, - $this->_bltID, - TRUE + $this->_bltID ); //create new soft-credit record, CRM-13981 @@ -1877,8 +1876,6 @@ protected function getSelectedMembershipLabels(): string { * * @param int $billingLocationID * ID of billing location type. - * @param bool $isRecur - * Is this recurring? * * @return \CRM_Contribute_DAO_Contribution * @@ -1892,8 +1889,7 @@ public static function processFormContribution( $contributionParams, $financialType, $online, - $billingLocationID, - $isRecur + $billingLocationID ) { $transaction = new CRM_Core_Transaction(); $contactID = $contributionParams['contact_id']; @@ -1914,7 +1910,7 @@ public static function processFormContribution( if (!isset($params['is_email_receipt']) && $isEmailReceipt) { $params['is_email_receipt'] = $isEmailReceipt; } - $params['is_recur'] = $isRecur; + $params['is_recur'] = TRUE; $params['payment_instrument_id'] = $contributionParams['payment_instrument_id'] ?? NULL; $recurringContributionID = CRM_Contribute_Form_Contribution_Confirm::processRecurringContribution($form, $params, $contactID, $financialType); From 7f2c42bf15d296e1f0d2d8227f3daa09faa5ff79 Mon Sep 17 00:00:00 2001 From: eileen Date: Wed, 16 Dec 2020 14:10:48 +1300 Subject: [PATCH 12/63] Remove duplicated tax assignments from copied code The assignment of tax data is happening twice on the form - once in generic code and once in code only reached for recurring contributions that was in previously-shared code. We can be fairly comfortable that in this latter case we don't need it as this is a marginal flow on this form whereas the main flow is being used 90% of the time & is doing the assignment --- CRM/Member/Form/Membership.php | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/CRM/Member/Form/Membership.php b/CRM/Member/Form/Membership.php index caa74005d93e..212b6d065983 100644 --- a/CRM/Member/Form/Membership.php +++ b/CRM/Member/Form/Membership.php @@ -1492,6 +1492,8 @@ public function submit() { $isRecur, $calcDates)); } + // This would always be true as we always add price set id into both + // quick config & non quick config price sets. if (!empty($lineItem[$this->_priceSetId])) { $invoicing = Civi::settings()->get('invoicing'); $taxAmount = FALSE; @@ -1938,29 +1940,6 @@ public static function processFormContribution( $contribution = CRM_Contribute_BAO_Contribution::add($contributionParams); - $invoiceSettings = Civi::settings()->get('contribution_invoice_settings'); - $invoicing = $invoiceSettings['invoicing'] ?? NULL; - if ($invoicing) { - $dataArray = []; - // @todo - interrogate the line items passed in on the params array. - // No reason to assume line items will be set on the form. - foreach ($form->_lineItem as $lineItemKey => $lineItemValue) { - foreach ($lineItemValue as $key => $value) { - if (isset($value['tax_amount']) && isset($value['tax_rate'])) { - if (isset($dataArray[$value['tax_rate']])) { - $dataArray[$value['tax_rate']] = $dataArray[$value['tax_rate']] + $value['tax_amount']; - } - else { - $dataArray[$value['tax_rate']] = $value['tax_amount']; - } - } - } - } - $smarty = CRM_Core_Smarty::singleton(); - $smarty->assign('dataArray', $dataArray); - $smarty->assign('totalTaxAmount', $params['tax_amount'] ?? NULL); - } - // lets store it in the form variable so postProcess hook can get to this and use it $form->_contributionID = $contribution->id; } From c497035a3eb6c48776da78cdcecefd1e2f2a2cfe Mon Sep 17 00:00:00 2001 From: eileen Date: Wed, 16 Dec 2020 14:03:12 +1300 Subject: [PATCH 13/63] Remove or hard-code variables from previously shared function This function was separated from the shared function for cleanup. This removes 3 variables - isRecur (always true) - billingId - this is elsewhere handled on the form - online (always false) For billing id this is handled through back office form shared functions. The handling might be slightly different but it seems more important that the form is internally consistent with how it creates billing addresses (between recur & non-recur) than with other forms --- CRM/Member/Form/Membership.php | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/CRM/Member/Form/Membership.php b/CRM/Member/Form/Membership.php index 5c1e1e6a1df6..ca0fe1d20807 100644 --- a/CRM/Member/Form/Membership.php +++ b/CRM/Member/Form/Membership.php @@ -1323,9 +1323,7 @@ public function submit() { 'thankyou_date' => $paymentParams['thankyou_date'] ?? NULL, 'payment_instrument_id' => $paymentInstrumentID, ], - $financialType, - FALSE, - $this->_bltID + $financialType ); //create new soft-credit record, CRM-13981 @@ -1871,11 +1869,6 @@ protected function getSelectedMembershipLabels(): string { * - thankyou_date (not all forms will set this) * * @param CRM_Financial_DAO_FinancialType $financialType - * @param bool $online - * Is the form a front end form? If so set a bunch of unpredictable things that should be passed in from the form. - * - * @param int $billingLocationID - * ID of billing location type. * * @return \CRM_Contribute_DAO_Contribution * @@ -1887,9 +1880,7 @@ public static function processFormContribution( $params, $result, $contributionParams, - $financialType, - $online, - $billingLocationID + $financialType ) { $transaction = new CRM_Core_Transaction(); $contactID = $contributionParams['contact_id']; @@ -1899,8 +1890,6 @@ public static function processFormContribution( // add these values for the recurringContrib function ,CRM-10188 $params['financial_type_id'] = $financialType->id; - $contributionParams['address_id'] = CRM_Contribute_BAO_Contribution::createAddress($params, $billingLocationID); - //@todo - this is being set from the form to resolve CRM-10188 - an // eNotice caused by it not being set @ the front end // however, we then get it being over-written with null for backend contributions @@ -1926,7 +1915,7 @@ public static function processFormContribution( $result, $receiptDate, $recurringContributionID), $contributionParams ); - $contributionParams['non_deductible_amount'] = CRM_Contribute_Form_Contribution_Confirm::getNonDeductibleAmount($params, $financialType, $online, $form); + $contributionParams['non_deductible_amount'] = CRM_Contribute_Form_Contribution_Confirm::getNonDeductibleAmount($params, $financialType, FALSE, $form); $contributionParams['skipCleanMoney'] = TRUE; // @todo this is the wrong place for this - it should be done as close to form submission // as possible @@ -1967,14 +1956,7 @@ public static function processFormContribution( //CRM-13981, processing honor contact into soft-credit contribution CRM_Contribute_BAO_ContributionSoft::processSoftContribution($params, $contribution); - if ($online && $contribution) { - CRM_Core_BAO_CustomValueTable::postProcess($params, - 'civicrm_contribution', - $contribution->id, - 'Contribution' - ); - } - elseif ($contribution) { + if ($contribution) { //handle custom data. $params['contribution_id'] = $contribution->id; if (!empty($params['custom']) && From 30836e925644f8348bf4ce7eb92199b99c1bffe7 Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Tue, 15 Dec 2020 19:15:55 -0800 Subject: [PATCH 14/63] composer.json - Update civicrm-cxn-rpc and phpseclib Before ------ Require `civicrm-cxn-rpc` v0.19 (with `phpseclib` v1.x) After ----- Require either of: * `civicrm-cxn-rpc` v0.20 (with `phpseclib` v2.x) * `civicrm-cxn-rpc` v0.19 (with `phpseclib` v1.x) Technical Details ----------------- * The public interfaces from civicrm-cxn-rpc are the same in 0.19+0.20. They only differ in which verison of `phpseclib` is used. * As pointed out in https://github.com/civicrm/civicrm-cxn-rpc/issues/9, we're not the only folks using phpseclib, so some flexibility on that seems good. * The primary change in phpseclib 2.x is the use of PHP namespaces (e.g. `Crypt_AES` => `\phpseclib\Crypt\AES`). * There are newer versions of both v0.19 and v0.20 which bundle an updated certificate. --- composer.json | 2 +- composer.lock | 68 ++++++++++++++++++++++++++++----------------------- 2 files changed, 38 insertions(+), 32 deletions(-) diff --git a/composer.json b/composer.json index 30130a8f0cb1..119c1bc7fe92 100644 --- a/composer.json +++ b/composer.json @@ -62,7 +62,7 @@ "marcj/topsort": "~1.1", "phpoffice/phpword": "^0.15.0", "pear/validate_finance_creditcard": "dev-master", - "civicrm/civicrm-cxn-rpc": "~0.19.01.08", + "civicrm/civicrm-cxn-rpc": "~0.20.12.01 || ~0.19.01.10", "pear/auth_sasl": "1.1.0", "pear/net_smtp": "1.9.*", "pear/net_socket": "1.0.*", diff --git a/composer.lock b/composer.lock index 5f9e7064a070..2442f8a46422 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f32350338fcd33fb89de9bcea7a49125", + "content-hash": "3877832b2ea8b061992a614b7a73a456", "packages": [ { "name": "adrienrn/php-mimetyper", @@ -258,21 +258,21 @@ }, { "name": "civicrm/civicrm-cxn-rpc", - "version": "v0.19.01.09", + "version": "v0.20.12.01", "source": { "type": "git", "url": "https://github.com/civicrm/civicrm-cxn-rpc.git", - "reference": "3ea668bc651adb4d61e96276f55e76ae22baea7a" + "reference": "b097258a642dc3e0dd9c264cb75b72d5274cac2f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/civicrm/civicrm-cxn-rpc/zipball/3ea668bc651adb4d61e96276f55e76ae22baea7a", - "reference": "3ea668bc651adb4d61e96276f55e76ae22baea7a", + "url": "https://api.github.com/repos/civicrm/civicrm-cxn-rpc/zipball/b097258a642dc3e0dd9c264cb75b72d5274cac2f", + "reference": "b097258a642dc3e0dd9c264cb75b72d5274cac2f", "shasum": "" }, "require": { - "phpseclib/phpseclib": "1.0.*", - "psr/log": "~1.0" + "phpseclib/phpseclib": "~2.0", + "psr/log": "~1.1" }, "type": "library", "autoload": { @@ -291,7 +291,7 @@ } ], "description": "RPC library for CiviConnect", - "time": "2020-02-05T03:24:26+00:00" + "time": "2020-12-16T02:35:45+00:00" }, { "name": "civicrm/composer-compile-lib", @@ -2040,50 +2040,42 @@ }, { "name": "phpseclib/phpseclib", - "version": "1.0.7", + "version": "2.0.29", "source": { "type": "git", "url": "https://github.com/phpseclib/phpseclib.git", - "reference": "0bb6c9b974cada100cad40f72ef186a199274f9b" + "reference": "497856a8d997f640b4a516062f84228a772a48a8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpseclib/phpseclib/zipball/0bb6c9b974cada100cad40f72ef186a199274f9b", - "reference": "0bb6c9b974cada100cad40f72ef186a199274f9b", + "url": "https://api.github.com/repos/phpseclib/phpseclib/zipball/497856a8d997f640b4a516062f84228a772a48a8", + "reference": "497856a8d997f640b4a516062f84228a772a48a8", "shasum": "" }, "require": { - "php": ">=5.0.0" + "php": ">=5.3.3" }, "require-dev": { "phing/phing": "~2.7", - "phpunit/phpunit": "~4.0", - "sami/sami": "~2.0", + "phpunit/phpunit": "^4.8.35|^5.7|^6.0", "squizlabs/php_codesniffer": "~2.0" }, "suggest": { "ext-gmp": "Install the GMP (GNU Multiple Precision) extension in order to speed up arbitrary precision integer arithmetic operations.", - "ext-mcrypt": "Install the Mcrypt extension in order to speed up a wide variety of cryptographic operations.", - "pear-pear/PHP_Compat": "Install PHP_Compat to get phpseclib working on PHP < 5.0.0." + "ext-libsodium": "SSH2/SFTP can make use of some algorithms provided by the libsodium-php extension.", + "ext-mcrypt": "Install the Mcrypt extension in order to speed up a few other cryptographic operations.", + "ext-openssl": "Install the OpenSSL extension in order to speed up a wide variety of cryptographic operations." }, "type": "library", "autoload": { - "psr-0": { - "Crypt": "phpseclib/", - "File": "phpseclib/", - "Math": "phpseclib/", - "Net": "phpseclib/", - "System": "phpseclib/" - }, "files": [ - "phpseclib/bootstrap.php", - "phpseclib/Crypt/Random.php" - ] + "phpseclib/bootstrap.php" + ], + "psr-4": { + "phpseclib\\": "phpseclib/" + } }, "notification-url": "https://packagist.org/downloads/", - "include-path": [ - "phpseclib/" - ], "license": [ "MIT" ], @@ -2135,7 +2127,21 @@ "x.509", "x509" ], - "time": "2017-06-05T06:30:30+00:00" + "funding": [ + { + "url": "https://github.com/terrafrost", + "type": "github" + }, + { + "url": "https://www.patreon.com/phpseclib", + "type": "patreon" + }, + { + "url": "https://tidelift.com/funding/github/packagist/phpseclib/phpseclib", + "type": "tidelift" + } + ], + "time": "2020-09-08T04:24:43+00:00" }, { "name": "psr/cache", From 1a3545fa626fbcf03de659be3ff27d8cc897b57d Mon Sep 17 00:00:00 2001 From: eileen Date: Wed, 16 Dec 2020 17:03:40 +1300 Subject: [PATCH 15/63] Convert previously shared function from static to non-static This no longer needs to be static as it is no longer shared with other forms (although it needs to be public to support the test class --- CRM/Contribute/Form/Contribution/Confirm.php | 11 ++++------- .../CRM/Contribute/Form/Contribution/ConfirmTest.php | 4 ++-- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/CRM/Contribute/Form/Contribution/Confirm.php b/CRM/Contribute/Form/Contribution/Confirm.php index 6cb6a92dcdaf..585813565f6d 100644 --- a/CRM/Contribute/Form/Contribution/Confirm.php +++ b/CRM/Contribute/Form/Contribution/Confirm.php @@ -1442,8 +1442,7 @@ protected function postProcessMembership( CRM_Price_BAO_LineItem::getLineItemArray($membershipParams); } - $paymentResult = self::processConfirm( - $form, + $paymentResult = $form->processConfirm( $membershipParams, $contactID, $financialTypeID, @@ -2301,7 +2300,7 @@ protected function processFormSubmission($contactID) { } } - $result = self::processConfirm($this, $paymentParams, + $result = $this->processConfirm($paymentParams, $contactID, $this->wrangleFinancialTypeID($this->_values['financial_type_id']), ($this->_mode == 'test') ? 1 : 0, @@ -2522,8 +2521,6 @@ protected static function isPaymentTransaction($form) { /** * Process payment after confirmation. * - * @param CRM_Core_Form $form - * Form object. * @param array $paymentParams * Array with payment related key. * value pairs @@ -2539,14 +2536,14 @@ protected static function isPaymentTransaction($form) { * @return array * associated array */ - public static function processConfirm( - &$form, + public function processConfirm( &$paymentParams, $contactID, $financialTypeID, $isTest, $isRecur ): array { + $form = $this; CRM_Core_Payment_Form::mapParams($form->_bltID, $form->_params, $paymentParams, TRUE); $isPaymentTransaction = self::isPaymentTransaction($form); diff --git a/tests/phpunit/CRM/Contribute/Form/Contribution/ConfirmTest.php b/tests/phpunit/CRM/Contribute/Form/Contribution/ConfirmTest.php index b802f33d7f84..0143f443ef40 100644 --- a/tests/phpunit/CRM/Contribute/Form/Contribution/ConfirmTest.php +++ b/tests/phpunit/CRM/Contribute/Form/Contribution/ConfirmTest.php @@ -105,7 +105,7 @@ public function testPayNowPayment() { 'skipLineItem' => 0, ]; - $processConfirmResult = CRM_Contribute_Form_Contribution_Confirm::processConfirm($form, + $processConfirmResult = $form->processConfirm( $form->_params, $contactID, $form->_values['financial_type_id'], @@ -155,7 +155,7 @@ public function testPayNowPayment() { 'relationship_type_id' => 5, 'is_current_employer' => 1, ]); - CRM_Contribute_Form_Contribution_Confirm::processConfirm($form, + $form->processConfirm( $form->_params, $form->_params['onbehalf_contact_id'], $form->_values['financial_type_id'], From 2ebe2e8cb3738b03f6342426e697a3aadfb14898 Mon Sep 17 00:00:00 2001 From: eileen Date: Wed, 16 Dec 2020 18:38:37 +1300 Subject: [PATCH 16/63] Only do cms account create from the one relevant place This function is called from 3 places - create CMS user is not applicable to the back office form and the other place actually blocks it... --- CRM/Contribute/Form/Contribution/Confirm.php | 29 ++++++++------------ 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/CRM/Contribute/Form/Contribution/Confirm.php b/CRM/Contribute/Form/Contribution/Confirm.php index 6cb6a92dcdaf..a4a4680c5eb8 100644 --- a/CRM/Contribute/Form/Contribution/Confirm.php +++ b/CRM/Contribute/Form/Contribution/Confirm.php @@ -1000,13 +1000,6 @@ public static function processFormContribution( CRM_Core_BAO_Note::add($noteParams, []); } - if (isset($params['related_contact'])) { - $contactID = $params['related_contact']; - } - elseif (isset($params['cms_contactID'])) { - $contactID = $params['cms_contactID']; - } - //create contribution activity w/ individual and target //activity w/ organisation contact id when onbelf, CRM-4027 $actParams = []; @@ -1025,12 +1018,6 @@ public static function processFormContribution( } $transaction->commit(); - // CRM-13074 - create the CMSUser after the transaction is completed as it - // is not appropriate to delete a valid contribution if a user create problem occurs - CRM_Contribute_BAO_Contribution_Utils::createCMSUser($params, - $contactID, - 'email-' . $billingLocationID - ); return $contribution; } @@ -1727,10 +1714,6 @@ protected function processSecondaryFinancialTransaction($contactID, &$form, $tem $form->set('membership_amount', $minimumFee); $form->assign('membership_amount', $minimumFee); - // we don't need to create the user twice, so lets disable cms_create_account - // irrespective of the value, CRM-2888 - $tempParams['cms_create_account'] = 0; - //set this variable as we are not creating pledge for //separate membership payment contribution. //so for differentiating membership contribution from @@ -2635,6 +2618,18 @@ public static function processConfirm( $form->_bltID, $isRecur ); + // CRM-13074 - create the CMSUser after the transaction is completed as it + // is not appropriate to delete a valid contribution if a user create problem occurs + if (isset($params['related_contact'])) { + $contactID = $params['related_contact']; + } + elseif (isset($params['cms_contactID'])) { + $contactID = $params['cms_contactID']; + } + CRM_Contribute_BAO_Contribution_Utils::createCMSUser($params, + $contactID, + 'email-' . $form->_bltID + ); $paymentParams['item_name'] = $form->_params['description']; From 014174e7a865aa20d371768a1a6eb9ea5df576bf Mon Sep 17 00:00:00 2001 From: Coleman Watts Date: Tue, 15 Dec 2020 19:37:31 -0500 Subject: [PATCH 17/63] Search kit: Rewrite input widget to support IN sets, relative dates, BETWEEN groups, etc. This deletes the crmSearchValue widget (and the now-empty crmSearchKit module), which had originally been copied from the API Explorer, and replaces it with a more flexible set of components with separate templates for each data type. --- ext/search/Civi/Search/Actions.php | 1 + ext/search/Civi/Search/Admin.php | 4 +- ext/search/ang/crmSearchActions.ang.php | 2 +- ext/search/ang/crmSearchActions.module.js | 16 ++- .../crmSearchActionDelete.html | 4 +- .../crmSearchActionUpdate.html | 6 +- .../crmSearchInput/boolean.html | 8 ++ .../crmMultiSelectDate.directive.js | 89 +++++++++++++ .../crmSearchInput.component.js | 51 ++++++++ .../crmSearchInput/crmSearchInput.html | 13 ++ .../crmSearchInputVal.component.js | 118 ++++++++++++++++++ .../crmSearchActions/crmSearchInput/date.html | 35 ++++++ .../crmSearchInput/entityRef.html | 6 + .../crmSearchInput/integer.html | 6 + .../crmSearchInput/select.html | 9 ++ .../crmSearchActions/crmSearchInput/text.html | 6 + ext/search/ang/crmSearchAdmin.ang.php | 2 +- .../crmSearchClause.component.js | 18 ++- .../ang/crmSearchAdmin/crmSearchClause.html | 2 +- ext/search/ang/crmSearchKit.ang.php | 14 --- ext/search/ang/crmSearchKit.module.js | 21 ---- .../crmSearchKit/crmSearchValue.directive.js | 113 ----------------- ext/search/css/search.css | 4 + 23 files changed, 386 insertions(+), 162 deletions(-) create mode 100644 ext/search/ang/crmSearchActions/crmSearchInput/boolean.html create mode 100644 ext/search/ang/crmSearchActions/crmSearchInput/crmMultiSelectDate.directive.js create mode 100644 ext/search/ang/crmSearchActions/crmSearchInput/crmSearchInput.component.js create mode 100644 ext/search/ang/crmSearchActions/crmSearchInput/crmSearchInput.html create mode 100644 ext/search/ang/crmSearchActions/crmSearchInput/crmSearchInputVal.component.js create mode 100644 ext/search/ang/crmSearchActions/crmSearchInput/date.html create mode 100644 ext/search/ang/crmSearchActions/crmSearchInput/entityRef.html create mode 100644 ext/search/ang/crmSearchActions/crmSearchInput/integer.html create mode 100644 ext/search/ang/crmSearchActions/crmSearchInput/select.html create mode 100644 ext/search/ang/crmSearchActions/crmSearchInput/text.html delete mode 100644 ext/search/ang/crmSearchKit.ang.php delete mode 100644 ext/search/ang/crmSearchKit.module.js delete mode 100644 ext/search/ang/crmSearchKit/crmSearchValue.directive.js diff --git a/ext/search/Civi/Search/Actions.php b/ext/search/Civi/Search/Actions.php index 5918d757dda1..c72a5970699e 100644 --- a/ext/search/Civi/Search/Actions.php +++ b/ext/search/Civi/Search/Actions.php @@ -24,6 +24,7 @@ public static function getActionSettings():array { return [ 'tasks' => self::getTasks(), 'groupOptions' => self::getGroupOptions(), + 'dateRanges' => \CRM_Utils_Array::makeNonAssociative(\CRM_Core_OptionGroup::values('relative_date_filters'), 'id', 'text'), ]; } diff --git a/ext/search/Civi/Search/Admin.php b/ext/search/Civi/Search/Admin.php index e7abc0e0ff64..6363b53c647b 100644 --- a/ext/search/Civi/Search/Admin.php +++ b/ext/search/Civi/Search/Admin.php @@ -43,8 +43,8 @@ public static function getOperators():array { '>=' => '≥', '<=' => '≤', 'CONTAINS' => ts('Contains'), - 'IN' => ts('Is In'), - 'NOT IN' => ts('Not In'), + 'IN' => ts('Is One Of'), + 'NOT IN' => ts('Not One Of'), 'LIKE' => ts('Is Like'), 'NOT LIKE' => ts('Not Like'), 'BETWEEN' => ts('Is Between'), diff --git a/ext/search/ang/crmSearchActions.ang.php b/ext/search/ang/crmSearchActions.ang.php index 478cd09609e2..6f62570aa282 100644 --- a/ext/search/ang/crmSearchActions.ang.php +++ b/ext/search/ang/crmSearchActions.ang.php @@ -10,7 +10,7 @@ 'ang/crmSearchActions', ], 'basePages' => [], - 'requires' => ['crmUi', 'crmUtil', 'dialogService', 'api4', 'crmSearchKit'], + 'requires' => ['crmUi', 'crmUtil', 'dialogService', 'api4', 'checklist-model'], 'settingsFactory' => ['\Civi\Search\Actions', 'getActionSettings'], 'permissions' => ['edit groups', 'administer reserved groups'], ]; diff --git a/ext/search/ang/crmSearchActions.module.js b/ext/search/ang/crmSearchActions.module.js index 912d2e54ff26..341e341287cf 100644 --- a/ext/search/ang/crmSearchActions.module.js +++ b/ext/search/ang/crmSearchActions.module.js @@ -2,6 +2,20 @@ "use strict"; // Declare module - angular.module('crmSearchActions', CRM.angRequires('crmSearchActions')); + angular.module('crmSearchActions', CRM.angRequires('crmSearchActions')) + + // Reformat an array of objects for compatibility with select2 + // Todo this probably belongs in core + .factory('formatForSelect2', function() { + return function(input, key, label, extra) { + return _.transform(input, function(result, item) { + var formatted = {id: item[key], text: item[label]}; + if (extra) { + _.merge(formatted, _.pick(item, extra)); + } + result.push(formatted); + }, []); + }; + }); })(angular, CRM.$, CRM._); diff --git a/ext/search/ang/crmSearchActions/crmSearchActionDelete.html b/ext/search/ang/crmSearchActions/crmSearchActionDelete.html index 855f3570d4f8..40d657cfef43 100644 --- a/ext/search/ang/crmSearchActions/crmSearchActionDelete.html +++ b/ext/search/ang/crmSearchActions/crmSearchActionDelete.html @@ -1,10 +1,10 @@
-
+

{{:: ts('Are you sure you want to delete %1 %2?', {1: model.ids.length, 2: $ctrl.entityTitle}) }}

-
+
diff --git a/ext/search/ang/crmSearchActions/crmSearchActionUpdate.html b/ext/search/ang/crmSearchActions/crmSearchActionUpdate.html index bb1a38033799..d3bae2a6bffe 100644 --- a/ext/search/ang/crmSearchActions/crmSearchActionUpdate.html +++ b/ext/search/ang/crmSearchActions/crmSearchActionUpdate.html @@ -1,9 +1,9 @@
-
+

{{:: ts('Update the %1 selected %2 with the following values:', {1: model.ids.length, 2: $ctrl.entityTitle}) }}

- +
@@ -13,5 +13,5 @@
-
+
diff --git a/ext/search/ang/crmSearchActions/crmSearchInput/boolean.html b/ext/search/ang/crmSearchActions/crmSearchInput/boolean.html new file mode 100644 index 000000000000..61ed9000e660 --- /dev/null +++ b/ext/search/ang/crmSearchActions/crmSearchInput/boolean.html @@ -0,0 +1,8 @@ +
+ + +
+
+ + +
diff --git a/ext/search/ang/crmSearchActions/crmSearchInput/crmMultiSelectDate.directive.js b/ext/search/ang/crmSearchActions/crmSearchInput/crmMultiSelectDate.directive.js new file mode 100644 index 000000000000..e830358fbbd2 --- /dev/null +++ b/ext/search/ang/crmSearchActions/crmSearchInput/crmMultiSelectDate.directive.js @@ -0,0 +1,89 @@ +(function(angular, $, _) { + "use strict"; + + angular.module('crmSearchActions') + .directive('crmMultiSelectDate', function () { + return { + restrict: 'A', + require: 'ngModel', + link: function (scope, element, attrs, ngModel) { + + var defaultDate = null; + + function getDisplayDate(date) { + return $.datepicker.formatDate(CRM.config.dateInputFormat, $.datepicker.parseDate('yy-mm-dd', date)); + } + + ngModel.$render = function () { + element.val(_.isArray(ngModel.$viewValue) ? ngModel.$viewValue.join(',') : ngModel.$viewValue).change(); + }; + + element + .crmSelect2({ + multiple: true, + data: [], + initSelection: function(element, callback) { + var values = []; + $.each($(element).val().split(','), function(k, v) { + values.push({ + text: getDisplayDate(v), + id: v + }); + }); + callback(values); + } + }) + .on('select2-opening', function(e) { + var $el = $(this), + $input = $('.select2-search-field input', $el.select2('container')); + // Prevent select2 from opening and show a datepicker instead + e.preventDefault(); + if (!$input.data('datepicker')) { + $input + .datepicker({ + beforeShow: function() { + var existingSelections = _.pluck($el.select2('data') || [], 'id'); + return { + changeMonth: true, + changeYear: true, + defaultDate: defaultDate, + beforeShowDay: function(date) { + // Don't allow the same date to be selected twice + var dateStr = $.datepicker.formatDate('yy-mm-dd', date); + if (_.includes(existingSelections, dateStr)) { + return [false, '', '']; + } + return [true, '', '']; + } + }; + } + }) + .datepicker('show') + .on('change.crmDate', function() { + if ($(this).val()) { + var data = $el.select2('data') || []; + defaultDate = $(this).datepicker('getDate'); + data.push({ + text: $.datepicker.formatDate(CRM.config.dateInputFormat, defaultDate), + id: $.datepicker.formatDate('yy-mm-dd', defaultDate) + }); + $el.select2('data', data, true); + } + }) + .on('keyup', function() { + $(this).val('').datepicker('show'); + }); + } + }) + // Don't leave datepicker open when clearing selections + .on('select2-removed', function() { + $('input.hasDatepicker', $(this).select2('container')) + .datepicker('hide'); + }) + .on('change', function() { + ngModel.$setViewValue(element.val().split(',')); + }); + } + }; + }); +})(angular, CRM.$, CRM._); diff --git a/ext/search/ang/crmSearchActions/crmSearchInput/crmSearchInput.component.js b/ext/search/ang/crmSearchActions/crmSearchInput/crmSearchInput.component.js new file mode 100644 index 000000000000..c37d99948a01 --- /dev/null +++ b/ext/search/ang/crmSearchActions/crmSearchInput/crmSearchInput.component.js @@ -0,0 +1,51 @@ +(function(angular, $, _) { + "use strict"; + + angular.module('crmSearchActions').component('crmSearchInput', { + bindings: { + field: '<', + 'op': '<', + 'format': '<', + 'optionKey': '<' + }, + require: {ngModel: 'ngModel'}, + templateUrl: '~/crmSearchActions/crmSearchInput/crmSearchInput.html', + controller: function($scope) { + var ts = $scope.ts = CRM.ts(), + ctrl = this; + + this.isMulti = function() { + // If there's a search operator, return `true` if the operator takes multiple values, else `false` + if (ctrl.op) { + return ctrl.op === 'IN' || ctrl.op === 'NOT IN'; + } + // If no search operator this is an input for e.g. the bulk update action + // Return `true` if the field is multi-valued, else `null` + return ctrl.field.serialize || ctrl.field.data_type === 'Array' ? true : null; + }; + + this.$onInit = function() { + + $scope.$watch('$ctrl.value', function() { + ctrl.ngModel.$setViewValue(ctrl.value); + }); + + // For the ON clause, string values must be quoted + ctrl.ngModel.$parsers.push(function(viewValue) { + return ctrl.format === 'json' && _.isString(viewValue) && viewValue.length ? JSON.stringify(viewValue) : viewValue; + }); + + // For the ON clause, unquote string values + ctrl.ngModel.$formatters.push(function(value) { + return ctrl.format === 'json' && _.isString(value) && value.length ? JSON.parse(value) : value; + }); + + this.ngModel.$render = function() { + ctrl.value = ctrl.ngModel.$viewValue; + }; + + }; + } + }); + +})(angular, CRM.$, CRM._); diff --git a/ext/search/ang/crmSearchActions/crmSearchInput/crmSearchInput.html b/ext/search/ang/crmSearchActions/crmSearchInput/crmSearchInput.html new file mode 100644 index 000000000000..bf434d9cf426 --- /dev/null +++ b/ext/search/ang/crmSearchActions/crmSearchInput/crmSearchInput.html @@ -0,0 +1,13 @@ +
+ +
+ + - + +
+ +
+ +
+ +
diff --git a/ext/search/ang/crmSearchActions/crmSearchInput/crmSearchInputVal.component.js b/ext/search/ang/crmSearchActions/crmSearchInput/crmSearchInputVal.component.js new file mode 100644 index 000000000000..6fbed45078d7 --- /dev/null +++ b/ext/search/ang/crmSearchActions/crmSearchInput/crmSearchInputVal.component.js @@ -0,0 +1,118 @@ +(function(angular, $, _) { + "use strict"; + + angular.module('crmSearchActions').component('crmSearchInputVal', { + bindings: { + field: '<', + 'multi': '<', + 'optionKey': '<' + }, + require: {ngModel: 'ngModel'}, + template: '
', + controller: function($scope, formatForSelect2) { + var ts = $scope.ts = CRM.ts(), + ctrl = this; + + this.$onInit = function() { + var rendered = false; + ctrl.dateRanges = CRM.crmSearchActions.dateRanges; + + this.ngModel.$render = function() { + ctrl.value = ctrl.ngModel.$viewValue; + if (!rendered && ctrl.field.input_type === 'Date') { + setDateType(); + } + rendered = true; + }; + + $scope.$watch('$ctrl.value', function() { + ctrl.ngModel.$setViewValue(ctrl.value); + }); + + function setDateType() { + if (_.findWhere(ctrl.dateRanges, {id: ctrl.value})) { + ctrl.dateType = 'range'; + } else if (ctrl.value === 'now') { + ctrl.dateType = 'now'; + } else if (_.includes(ctrl.value, 'now -')) { + ctrl.dateType = 'now -'; + } else if (_.includes(ctrl.value, 'now +')) { + ctrl.dateType = 'now +'; + } else { + ctrl.dateType = 'fixed'; + } + } + }; + + this.changeDateType = function() { + switch (ctrl.dateType) { + case 'fixed': + ctrl.value = ''; + break; + + case 'range': + ctrl.value = ctrl.dateRanges[0].id; + break; + + case 'now': + ctrl.value = 'now'; + break; + + default: + ctrl.value = ctrl.dateType + ' 1 day'; + } + }; + + this.dateUnits = function(setUnit) { + var vals = ctrl.value.split(' '); + if (arguments.length) { + vals[3] = setUnit; + ctrl.value = vals.join(' '); + } else { + return vals[3]; + } + }; + + this.dateNumber = function(setNumber) { + var vals = ctrl.value.split(' '); + if (arguments.length) { + vals[2] = setNumber; + ctrl.value = vals.join(' '); + } else { + return parseInt(vals[2], 10); + } + }; + + this.getTemplate = function() { + + if (ctrl.field.input_type === 'Date') { + return '~/crmSearchActions/crmSearchInput/date.html'; + } + + if (ctrl.field.data_type === 'Boolean') { + return '~/crmSearchActions/crmSearchInput/boolean.html'; + } + + if (ctrl.field.options) { + return '~/crmSearchActions/crmSearchInput/select.html'; + } + + if (ctrl.field.fk_entity) { + return '~/crmSearchActions/crmSearchInput/entityRef.html'; + } + + if (ctrl.field.data_type === 'Integer') { + return '~/crmSearchActions/crmSearchInput/integer.html'; + } + + return '~/crmSearchActions/crmSearchInput/text.html'; + }; + + this.getFieldOptions = function() { + return {results: formatForSelect2(ctrl.field.options, ctrl.optionKey || 'id', 'label', ['description', 'color', 'icon'])}; + }; + + } + }); + +})(angular, CRM.$, CRM._); diff --git a/ext/search/ang/crmSearchActions/crmSearchInput/date.html b/ext/search/ang/crmSearchActions/crmSearchInput/date.html new file mode 100644 index 000000000000..78e4d03d6d66 --- /dev/null +++ b/ext/search/ang/crmSearchActions/crmSearchInput/date.html @@ -0,0 +1,35 @@ +
+ + +
+ +
+ + +
+ +
+ +
+ +
+ + +
+ +
+
diff --git a/ext/search/ang/crmSearchActions/crmSearchInput/entityRef.html b/ext/search/ang/crmSearchActions/crmSearchInput/entityRef.html new file mode 100644 index 000000000000..b0050d7421ee --- /dev/null +++ b/ext/search/ang/crmSearchActions/crmSearchInput/entityRef.html @@ -0,0 +1,6 @@ +
+ +
+
+ +
diff --git a/ext/search/ang/crmSearchActions/crmSearchInput/integer.html b/ext/search/ang/crmSearchActions/crmSearchInput/integer.html new file mode 100644 index 000000000000..6c0f01ace6ac --- /dev/null +++ b/ext/search/ang/crmSearchActions/crmSearchInput/integer.html @@ -0,0 +1,6 @@ +
+ +
+
+ +
diff --git a/ext/search/ang/crmSearchActions/crmSearchInput/select.html b/ext/search/ang/crmSearchActions/crmSearchInput/select.html new file mode 100644 index 000000000000..464c92e43171 --- /dev/null +++ b/ext/search/ang/crmSearchActions/crmSearchInput/select.html @@ -0,0 +1,9 @@ +
+ +
+
+ +
+
+ +
diff --git a/ext/search/ang/crmSearchActions/crmSearchInput/text.html b/ext/search/ang/crmSearchActions/crmSearchInput/text.html new file mode 100644 index 000000000000..61a4391c48d5 --- /dev/null +++ b/ext/search/ang/crmSearchActions/crmSearchInput/text.html @@ -0,0 +1,6 @@ +
+ +
+
+ +
diff --git a/ext/search/ang/crmSearchAdmin.ang.php b/ext/search/ang/crmSearchAdmin.ang.php index a947d51b3009..9d5ee8e52132 100644 --- a/ext/search/ang/crmSearchAdmin.ang.php +++ b/ext/search/ang/crmSearchAdmin.ang.php @@ -14,6 +14,6 @@ ], 'bundles' => ['bootstrap3'], 'basePages' => ['civicrm/admin/search'], - 'requires' => ['crmUi', 'crmUtil', 'ngRoute', 'ui.sortable', 'ui.bootstrap', 'api4', 'crmSearchActions', 'crmSearchKit', 'crmRouteBinder'], + 'requires' => ['crmUi', 'crmUtil', 'ngRoute', 'ui.sortable', 'ui.bootstrap', 'api4', 'crmSearchActions', 'crmRouteBinder'], 'settingsFactory' => ['\Civi\Search\Admin', 'getAdminSettings'], ]; diff --git a/ext/search/ang/crmSearchAdmin/crmSearchClause.component.js b/ext/search/ang/crmSearchAdmin/crmSearchClause.component.js index 3c196a440b10..de5ce60e43d8 100644 --- a/ext/search/ang/crmSearchAdmin/crmSearchClause.component.js +++ b/ext/search/ang/crmSearchAdmin/crmSearchClause.component.js @@ -83,12 +83,24 @@ } }; - // Add/remove value if operator allows for one this.changeClauseOperator = function(clause) { + // Add/remove value if operator allows for one if (_.contains(clause[1], 'NULL')) { clause.length = 2; - } else if (clause.length === 2) { - clause.push(''); + } else { + if (clause.length === 2) { + clause.push(''); + } + // Change multi/single value to/from an array + var shouldBeArray = (clause[1] === 'IN' || clause[1] === 'NOT IN' || clause[1] === 'BETWEEN' || clause[1] === 'NOT BETWEEN'); + if (!_.isArray(clause[2]) && shouldBeArray) { + clause[2] = []; + } else if (_.isArray(clause[2]) && !shouldBeArray) { + clause[2] = ''; + } + if (clause[1] === 'BETWEEN' || clause[1] === 'NOT BETWEEN') { + clause[2].length = 2; + } } }; diff --git a/ext/search/ang/crmSearchAdmin/crmSearchClause.html b/ext/search/ang/crmSearchAdmin/crmSearchClause.html index bcc23f7038f0..80480c76b1da 100644 --- a/ext/search/ang/crmSearchAdmin/crmSearchClause.html +++ b/ext/search/ang/crmSearchAdmin/crmSearchClause.html @@ -17,7 +17,7 @@
- +
diff --git a/ext/search/ang/crmSearchKit.ang.php b/ext/search/ang/crmSearchKit.ang.php deleted file mode 100644 index 1827d07ce898..000000000000 --- a/ext/search/ang/crmSearchKit.ang.php +++ /dev/null @@ -1,14 +0,0 @@ - [ - 'ang/crmSearchKit.module.js', - 'ang/crmSearchKit/*.js', - 'ang/crmSearchKit/*/*.js', - ], - 'partials' => [ - 'ang/crmSearchKit', - ], - 'basePages' => [], - 'requires' => [], -]; diff --git a/ext/search/ang/crmSearchKit.module.js b/ext/search/ang/crmSearchKit.module.js deleted file mode 100644 index 76caa98e61a0..000000000000 --- a/ext/search/ang/crmSearchKit.module.js +++ /dev/null @@ -1,21 +0,0 @@ -(function(angular, $, _) { - "use strict"; - - // Declare module - angular.module('crmSearchKit', CRM.angRequires('crmSearchKit')) - - // Reformat an array of objects for compatibility with select2 - // Todo this probably belongs in core - .factory('formatForSelect2', function() { - return function(input, key, label, extra) { - return _.transform(input, function(result, item) { - var formatted = {id: item[key], text: item[label]}; - if (extra) { - _.merge(formatted, _.pick(item, extra)); - } - result.push(formatted); - }, []); - }; - }); - -})(angular, CRM.$, CRM._); diff --git a/ext/search/ang/crmSearchKit/crmSearchValue.directive.js b/ext/search/ang/crmSearchKit/crmSearchValue.directive.js deleted file mode 100644 index 2d1bbd09e5de..000000000000 --- a/ext/search/ang/crmSearchKit/crmSearchValue.directive.js +++ /dev/null @@ -1,113 +0,0 @@ -(function(angular, $, _) { - "use strict"; - - angular.module('crmSearchKit').directive('crmSearchValue', function($interval, formatForSelect2) { - return { - scope: { - data: '=crmSearchValue' - }, - require: 'ngModel', - link: function (scope, element, attrs, ngModel) { - var ts = scope.ts = CRM.ts(), - multi = _.includes(['IN', 'NOT IN', 'BETWEEN', 'NOT BETWEEN'], scope.data.op), - format = scope.data.format; - - function destroyWidget() { - var $el = $(element); - if ($el.is('.crm-form-date-wrapper .crm-hidden-date')) { - $el.crmDatepicker('destroy'); - } - if ($el.is('.select2-container + input')) { - $el.crmEntityRef('destroy'); - } - $(element).removeData().removeAttr('type').removeAttr('placeholder').show(); - } - - function makeWidget(field, op, optionKey) { - var $el = $(element), - inputType = field.input_type, - dataType = field.data_type; - if (!op) { - op = field.serialize || dataType === 'Array' ? 'IN' : '='; - } - multi = _.includes(['IN', 'NOT IN', 'BETWEEN', 'NOT BETWEEN'], op); - if (op === 'IS NULL' || op === 'IS NOT NULL') { - $el.hide(); - return; - } - if (inputType === 'Date') { - if (_.includes(['=', '!=', '>', '>=', '<', '<='], op)) { - $el.crmDatepicker({time: (field.input_attrs && field.input_attrs.time) || false}); - } - } else if (_.includes(['=', '!=', 'IN', 'NOT IN', 'CONTAINS'], op) && (field.fk_entity || field.options || dataType === 'Boolean')) { - if (field.options) { - if (field.options === true) { - $el.addClass('loading'); - var waitForOptions = $interval(function() { - if (field.options !== true) { - $interval.cancel(waitForOptions); - $el.removeClass('loading').crmSelect2({data: getFieldOptions, multiple: multi}); - } - }, 200); - } - $el.attr('placeholder', ts('select')).crmSelect2({data: getFieldOptions, multiple: multi}); - } else if (field.fk_entity) { - $el.crmEntityRef({entity: field.fk_entity, select:{multiple: multi}}); - } else if (dataType === 'Boolean') { - $el.attr('placeholder', ts('- select -')).crmSelect2({allowClear: false, multiple: multi, placeholder: ts('- select -'), data: [ - // FIXME: it would be more correct to use real true/false booleans instead of numbers, but select2 doesn't seem to like them - {id: 1, text: ts('Yes')}, - {id: 0, text: ts('No')} - ]}); - } - } else if (dataType === 'Integer' && !multi) { - $el.attr('type', 'number'); - } - - function getFieldOptions() { - return {results: formatForSelect2(field.options, optionKey, 'label', ['description', 'color', 'icon'])}; - } - } - - // Copied from ng-list but applied conditionally if field is multi-valued - var parseList = function(viewValue) { - // If the viewValue is invalid (say required but empty) it will be `undefined` - if (_.isUndefined(viewValue)) return; - - if (!multi) { - return format === 'json' ? JSON.stringify(viewValue) : viewValue; - } - - var list = []; - - if (viewValue) { - _.each(viewValue.split(','), function(value) { - if (value) list.push(_.trim(value)); - }); - } - - return list; - }; - - // Copied from ng-list - ngModel.$parsers.push(parseList); - ngModel.$formatters.push(function(value) { - return _.isArray(value) ? value.join(', ') : (format === 'json' && value !== '' ? JSON.parse(value) : value); - }); - - // Copied from ng-list - ngModel.$isEmpty = function(value) { - return !value || !value.length; - }; - - scope.$watchCollection('data', function(data) { - destroyWidget(); - if (data.field) { - makeWidget(data.field, data.op, data.optionKey || 'id'); - } - }); - } - }; - }); - -})(angular, CRM.$, CRM._); diff --git a/ext/search/css/search.css b/ext/search/css/search.css index cfcccfd27074..6e41dbc9c952 100644 --- a/ext/search/css/search.css +++ b/ext/search/css/search.css @@ -126,6 +126,10 @@ width: 110px; } +#bootstrap-theme.crm-search input[type=number] { + width: 90px; +} + #bootstrap-theme.crm-search .api4-add-where-group-menu { min-width: 80px; background-color: rgba(186, 225, 251, 0.94); From 88b95fd870ec3c01cd95e233f0f610e827955c46 Mon Sep 17 00:00:00 2001 From: Coleman Watts Date: Wed, 16 Dec 2020 17:27:58 -0500 Subject: [PATCH 18/63] Add min-width to flex columns for responsive layout on small screens The .crm-flex-box class is new and only used in 2 places: Search Kit & the Dashboard. This sets a min-width on those layouts so the 2 columns collapse to 1 on small screens. --- css/civicrm.css | 1 + css/dashboard.css | 7 +++++++ ext/search/ang/crmSearchAdmin/compose/criteria.html | 4 ++-- ext/search/css/search.css | 4 ++++ 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/css/civicrm.css b/css/civicrm.css index 722d5588ee5f..519289821c46 100644 --- a/css/civicrm.css +++ b/css/civicrm.css @@ -21,6 +21,7 @@ .crm-container .crm-flex-box { display: flex; + flex-wrap: wrap; } .crm-container .crm-flex-box > * { flex: 1; diff --git a/css/dashboard.css b/css/dashboard.css index 88296b1be7a8..2586b3c9012d 100644 --- a/css/dashboard.css +++ b/css/dashboard.css @@ -2,6 +2,13 @@ min-height: 200px; } +#civicrm-dashboard > .crm-flex-box > .crm-flex-2 { + min-width: 300px; +} +#civicrm-dashboard > .crm-flex-box > .crm-flex-3 { + min-width: 450px; +} + .crm-container .crm-dashlet { margin: 10px; box-shadow: 1px 1px 4px 1px rgba(0,0,0,0.2); diff --git a/ext/search/ang/crmSearchAdmin/compose/criteria.html b/ext/search/ang/crmSearchAdmin/compose/criteria.html index 0f7658182655..78c75a391a47 100644 --- a/ext/search/ang/crmSearchAdmin/compose/criteria.html +++ b/ext/search/ang/crmSearchAdmin/compose/criteria.html @@ -1,5 +1,5 @@
-
+
@@ -36,7 +36,7 @@
-
+
diff --git a/ext/search/css/search.css b/ext/search/css/search.css index cfcccfd27074..ac01597cd03b 100644 --- a/ext/search/css/search.css +++ b/ext/search/css/search.css @@ -1,3 +1,7 @@ +#bootstrap-theme .crm-search-criteria-column { + min-width: 500px; +} + #bootstrap-theme #crm-search-results-page-size { width: 5em; } From 3068cf0f945471e7777e1dc9858eba00c6b13ee9 Mon Sep 17 00:00:00 2001 From: Coleman Watts Date: Thu, 17 Dec 2020 16:15:11 -0500 Subject: [PATCH 19/63] APIv4: Normalize option list descriptions as plain text Our schema is inconsistent about whether `description` fields allow html, but it's usually assumed to be plain text, so we strip_tags() to standardize it. --- Civi/Api4/Service/Spec/FieldSpec.php | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Civi/Api4/Service/Spec/FieldSpec.php b/Civi/Api4/Service/Spec/FieldSpec.php index 73451ddee126..0d503ad748b2 100644 --- a/Civi/Api4/Service/Spec/FieldSpec.php +++ b/Civi/Api4/Service/Spec/FieldSpec.php @@ -435,7 +435,9 @@ public function getOptions($values = [], $return = TRUE) { } /** - * Supplement the data from + * Augment the 2 values returned by BAO::buildOptions (id, label) with extra properties (name, description, color, icon, etc). + * + * We start with BAO::buildOptions in order to respect hooks which may be adding/removing items, then we add the extra data. * * @param \CRM_Core_DAO $baoName * @param string $fieldName @@ -470,7 +472,9 @@ private function addOptionProps($baoName, $fieldName, $values, $return) { foreach ($extraStuff as $item) { if (isset($optionIndex[$item[$keyColumn]])) { foreach ($return as $ret) { - $this->options[$optionIndex[$item[$keyColumn]]][$ret] = $item[$ret] ?? NULL; + // Note: our schema is inconsistent about whether `description` fields allow html, + // but it's usually assumed to be plain text, so we strip_tags() to standardize it. + $this->options[$optionIndex[$item[$keyColumn]]][$ret] = ($ret === 'description' && isset($item[$ret])) ? strip_tags($item[$ret]) : $item[$ret] ?? NULL; } } } @@ -488,7 +492,9 @@ private function addOptionProps($baoName, $fieldName, $values, $return) { while ($query->fetch()) { foreach ($return as $ret) { if (property_exists($query, $ret)) { - $this->options[$optionIndex[$query->id]][$ret] = $query->$ret; + // Note: our schema is inconsistent about whether `description` fields allow html, + // but it's usually assumed to be plain text, so we strip_tags() to standardize it. + $this->options[$optionIndex[$query->id]][$ret] = $ret === 'description' ? strip_tags($query->$ret) : $query->$ret; } } } From eb9fa017b4d8f02c8e9f838bdad3ef90dad0254c Mon Sep 17 00:00:00 2001 From: Jitendra Purohit Date: Fri, 18 Dec 2020 10:13:03 +0530 Subject: [PATCH 20/63] Fix pledge on contribution page --- CRM/Contribute/Form/ContributionPage/Amount.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CRM/Contribute/Form/ContributionPage/Amount.php b/CRM/Contribute/Form/ContributionPage/Amount.php index 16fdf6324518..7aca7aac4fa7 100644 --- a/CRM/Contribute/Form/ContributionPage/Amount.php +++ b/CRM/Contribute/Form/ContributionPage/Amount.php @@ -752,7 +752,7 @@ public function postProcess() { $deletePledgeBlk = FALSE; $pledgeBlockParams = [ 'entity_id' => $contributionPageID, - 'entity_table' => ts('civicrm_contribution_page'), + 'entity_table' => 'civicrm_contribution_page', ]; if ($this->_pledgeBlockID) { $pledgeBlockParams['id'] = $this->_pledgeBlockID; From 08b2ba0ca7cc8a5cac9d0cb50eb9cb0edce4f245 Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Fri, 18 Dec 2020 18:03:21 -0800 Subject: [PATCH 21/63] CRM_Utils_String - Add URL-style base64 codec --- CRM/Utils/String.php | 25 +++++++++++++++++++++++++ tests/phpunit/CRM/Utils/StringTest.php | 13 +++++++++++++ 2 files changed, 38 insertions(+) diff --git a/CRM/Utils/String.php b/CRM/Utils/String.php index 7c0897436508..8e761a5be823 100644 --- a/CRM/Utils/String.php +++ b/CRM/Utils/String.php @@ -221,6 +221,31 @@ public static function isAscii($str, $utf8 = TRUE) { } } + /** + * Encode string using URL-safe Base64. + * + * @param string $v + * + * @return string + * @see https://tools.ietf.org/html/rfc4648#section-5 + */ + public static function base64UrlEncode($v) { + return rtrim(str_replace(['+', '/'], ['-', '_'], base64_encode($v)), '='); + } + + /** + * Decode string using URL-safe Base64. + * + * @param string $v + * + * @return false|string + * @see https://tools.ietf.org/html/rfc4648#section-5 + */ + public static function base64UrlDecode($v) { + // PHP base64_decode() is already forgiving about padding ("="). + return base64_decode(str_replace(['-', '_'], ['+', '/'], $v)); + } + /** * Determine the string replacements for redaction. * on the basis of the regular expressions diff --git a/tests/phpunit/CRM/Utils/StringTest.php b/tests/phpunit/CRM/Utils/StringTest.php index be31cfa8d02c..dcc461568ced 100644 --- a/tests/phpunit/CRM/Utils/StringTest.php +++ b/tests/phpunit/CRM/Utils/StringTest.php @@ -10,6 +10,19 @@ public function setUp() { parent::setUp(); } + public function testBase64Url() { + $examples = [ + 'a' => 'YQ', + 'ab' => 'YWI', + 'abc' => 'YWJj', + '3f>' => 'M2Y-', + ]; + foreach ($examples as $raw => $b64) { + $this->assertEquals($b64, CRM_Utils_String::base64UrlEncode($raw)); + $this->assertEquals($raw, CRM_Utils_String::base64UrlDecode($b64)); + } + } + public function testStripPathChars() { $testSet = [ '' => '', From 281eacd865bdf56317eb58eb13076b3e4c33e032 Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Wed, 16 Dec 2020 13:26:32 -0800 Subject: [PATCH 22/63] (dev/core#2258) CryptoRegistry - Keep track of available keys+suites. Hookable. --- CRM/Utils/Hook.php | 17 ++ Civi/Core/Container.php | 45 ++++ Civi/Crypto/CipherSuiteInterface.php | 51 ++++ Civi/Crypto/CryptoRegistry.php | 244 ++++++++++++++++++ Civi/Crypto/Exception/CryptoException.php | 9 + .../Civi/Crypto/CryptoRegistryTest.php | 109 ++++++++ tests/phpunit/Civi/Crypto/CryptoTestTrait.php | 63 +++++ 7 files changed, 538 insertions(+) create mode 100644 Civi/Crypto/CipherSuiteInterface.php create mode 100644 Civi/Crypto/CryptoRegistry.php create mode 100644 Civi/Crypto/Exception/CryptoException.php create mode 100644 tests/phpunit/Civi/Crypto/CryptoRegistryTest.php create mode 100644 tests/phpunit/Civi/Crypto/CryptoTestTrait.php diff --git a/CRM/Utils/Hook.php b/CRM/Utils/Hook.php index 95a5496ffd60..8ec1d7235da5 100644 --- a/CRM/Utils/Hook.php +++ b/CRM/Utils/Hook.php @@ -1759,6 +1759,23 @@ public static function alterTemplateFile($formName, &$form, $context, &$tplName) ); } + /** + * Initialize the cryptographic service. + * + * This may be used to register additional keys or cipher-suites. + * + * @param \Civi\Crypto\CryptoRegistry $crypto + * + * @return mixed + */ + public static function crypto($crypto) { + return self::singleton()->invoke(['crypto'], $crypto, self::$_nullObject, + self::$_nullObject, self::$_nullObject, self::$_nullObject, + self::$_nullObject, + 'civicrm_crypto' + ); + } + /** * This hook collects the trigger definition from all components. * diff --git a/Civi/Core/Container.php b/Civi/Core/Container.php index 4f38b4e3a2f0..404405adadb9 100644 --- a/Civi/Core/Container.php +++ b/Civi/Core/Container.php @@ -218,6 +218,9 @@ public function createContainer() { $container->setDefinition('pear_mail', new Definition('Mail')) ->setFactory('CRM_Utils_Mail::createMailer')->setPublic(TRUE); + $container->setDefinition('crypto.registry', new Definition('Civi\Crypto\CryptoService')) + ->setFactory(__CLASS__ . '::createCryptoRegistry')->setPublic(TRUE); + if (empty(\Civi::$statics[__CLASS__]['boot'])) { throw new \RuntimeException('Cannot initialize container. Boot services are undefined.'); } @@ -499,6 +502,48 @@ public static function createCacheConfig() { return new \ArrayObject($settings); } + /** + * Initialize the cryptogrpahic registry. It tracks available ciphers and keys. + * + * @return \Civi\Crypto\CryptoRegistry + * @throws \CRM_Core_Exception + * @throws \Civi\Crypto\Exception\CryptoException + */ + public static function createCryptoRegistry() { + $crypto = new \Civi\Crypto\CryptoRegistry(); + + $crypto->addPlainText(['tags' => ['CRED']]); + if (defined('CIVICRM_CRED_KEYS')) { + foreach (explode(' ', CIVICRM_CRED_KEYS) as $n => $keyExpr) { + $crypto->addSymmetricKey($crypto->parseKey($keyExpr) + [ + 'tags' => ['CRED'], + 'weight' => $n, + ]); + } + } + if (defined('CIVICRM_SITE_KEY')) { + // Recent upgrades may not have CIVICRM_CRED_KEYS. Transitional support - the CIVICRM_SITE_KEY is last-priority option for credentials. + $crypto->addSymmetricKey([ + 'key' => hash_hkdf('sha256', CIVICRM_SITE_KEY), + 'suite' => 'aes-cbc', + 'tags' => ['CRED'], + 'weight' => 30000, + ]); + } + //if (isset($_COOKIE['CIVICRM_FORM_KEY'])) { + // $crypto->addSymmetricKey([ + // 'key' => base64_decode($_COOKIE['CIVICRM_FORM_KEY']), + // 'suite' => 'aes-cbc', + // 'tag' => ['FORM'], + // ]); + // // else: somewhere in CRM_Core_Form, we may need to initialize CIVICRM_FORM_KEY + //} + + // Allow plugins to add/replace any keys and ciphers. + \CRM_Utils_Hook::crypto($crypto); + return $crypto; + } + /** * Get a list of boot services. * diff --git a/Civi/Crypto/CipherSuiteInterface.php b/Civi/Crypto/CipherSuiteInterface.php new file mode 100644 index 000000000000..925e743d3146 --- /dev/null +++ b/Civi/Crypto/CipherSuiteInterface.php @@ -0,0 +1,51 @@ +cipherSuites['plain'] = TRUE; + $this->keys['plain'] = [ + 'key' => '', + 'suite' => 'plain', + 'tags' => [], + 'id' => 'plain', + 'weight' => self::LAST_WEIGHT, + ]; + + // Base64 - Useful for precise control. Relatively quick decode. Please bring your own entropy. + $this->kdfs['b64'] = 'base64_decode'; + + // HKDF - Forgiving about diverse inputs. Relatively quick decode. Please bring your own entropy. + $this->kdfs['hkdf-sha256'] = function($v) { + // NOTE: 256-bit output by default. Useful for pairing with AES-256. + return hash_hkdf('sha256', $v); + }; + + // Possible future options: Read from PEM file. Run PBKDF2 on a passphrase. + } + + /** + * @param string|array $options + * Additional options: + * - key: string, a representation of the key as binary + * - suite: string, ex: 'aes-cbc' + * - tags: string[] + * - weight: int, default 0 + * - id: string, a unique identifier for this key. (default: fingerprint the key+suite) + * + * @return array + * The full key record. (Same format as $options) + * @throws \Civi\Crypto\Exception\CryptoException + */ + public function addSymmetricKey($options) { + $defaults = [ + 'suite' => self::DEFAULT_SUITE, + 'weight' => 0, + ]; + $options = array_merge($defaults, $options); + + if (!isset($options['key'])) { + throw new CryptoException("Missing crypto key"); + } + + if (!isset($options['id'])) { + $options['id'] = \CRM_Utils_String::base64UrlEncode(sha1($options['suite'] . chr(0) . $options['key'], TRUE)); + } + // Manual key IDs should be validated. + elseif (!$this->isValidKeyId($options['id'])) { + throw new CryptoException("Malformed key ID"); + } + + $this->keys[$options['id']] = $options; + return $options; + } + + /** + * Determine if a key ID is well-formed. + * + * @param string $id + * @return bool + */ + public function isValidKeyId($id) { + if (strpos($id, "\n") !== FALSE) { + return FALSE; + } + return (bool) preg_match(';^[a-zA-Z0-9_\-\.:,=+/\;\\\\]+$;s', $id); + } + + /** + * Enable plain-text encoding. + * + * @param array $options + * Array with options: + * - tags: string[] + * @return array + */ + public function addPlainText($options) { + if (!isset($this->keys['plain'])) { + } + if (isset($options['tags'])) { + $this->keys['plain']['tags'] = array_merge( + $options['tags'] + ); + } + return $this->keys['plain']; + } + + /** + * @param CipherSuiteInterface $cipherSuite + * The encryption/decryption callback/handler + * @param string[]|NULL $names + * Symbolic names. Ex: 'aes-cbc' + * If NULL, probe $cipherSuite->getNames() + */ + public function addCipherSuite(CipherSuiteInterface $cipherSuite, $names = NULL) { + $names = $names ?: $cipherSuite->getSuites(); + foreach ($names as $name) { + $this->cipherSuites[$name] = $cipherSuite; + } + } + + public function getKeys() { + return $this->keys; + } + + /** + * Locate a key in the list of available keys. + * + * @param string|string[] $keyIds + * List of IDs or tags. The first match in the list is returned. + * If multiple keys match the same tag, then the one with lowest 'weight' is returned. + * @return array + * @throws \Civi\Crypto\Exception\CryptoException + */ + public function findKey($keyIds) { + $keyIds = (array) $keyIds; + foreach ($keyIds as $keyIdOrTag) { + if (isset($this->keys[$keyIdOrTag])) { + return $this->keys[$keyIdOrTag]; + } + + $matchKeyId = NULL; + $matchWeight = self::LAST_WEIGHT; + foreach ($this->keys as $key) { + if (in_array($keyIdOrTag, $key['tags']) && $key['weight'] <= $matchWeight) { + $matchKeyId = $key['id']; + $matchWeight = $key['weight']; + } + } + if ($matchKeyId !== NULL) { + return $this->keys[$matchKeyId]; + } + } + + throw new CryptoException("Failed to find key by ID or tag (" . implode(' ', $keyIds) . ")"); + } + + /** + * @param string $name + * @return \Civi\Crypto\CipherSuiteInterface + * @throws \Civi\Crypto\Exception\CryptoException + */ + public function findSuite($name) { + if (isset($this->cipherSuites[$name])) { + return $this->cipherSuites[$name]; + } + else { + throw new CryptoException('Unknown cipher suite ' . $name); + } + } + + /** + * @param string $keyExpr + * String in the form "::". + * + * 'aes-cbc:b64:cGxlYXNlIHVzZSAzMiBieXRlcyBmb3IgYWVzLTI1NiE=' + * 'aes-cbc:hkdf-sha256:ABCD1234ABCD1234ABCD1234ABCD1234' + * '::ABCD1234ABCD1234ABCD1234ABCD1234' + * + * @return array + * Properties: + * - key: string, binary representation + * - suite: string, ex: 'aes-cbc' + * @throws CryptoException + */ + public function parseKey($keyExpr) { + list($suite, $keyFunc, $keyVal) = explode(':', $keyExpr); + if ($suite === '') { + $suite = self::DEFAULT_SUITE; + } + if ($keyFunc === '') { + $keyFunc = self::DEFAULT_KDF; + } + if (isset($this->kdfs[$keyFunc])) { + return [ + 'suite' => $suite, + 'key' => call_user_func($this->kdfs[$keyFunc], $keyVal), + ]; + } + else { + throw new CryptoException("Crypto key has unrecognized type"); + } + } + +} diff --git a/Civi/Crypto/Exception/CryptoException.php b/Civi/Crypto/Exception/CryptoException.php new file mode 100644 index 000000000000..84677ace41ec --- /dev/null +++ b/Civi/Crypto/Exception/CryptoException.php @@ -0,0 +1,9 @@ +setHook('civicrm_crypto', [$this, 'registerExampleKeys']); + } + + public function testParseKey() { + $examples = self::getExampleKeys(); + $registry = \Civi::service('crypto.registry'); + + $key0 = $registry->parseKey($examples[0]); + $this->assertEquals("please use 32 bytes for aes-256!", $key0['key']); + $this->assertEquals('aes-cbc', $key0['suite']); + + $key1 = $registry->parseKey($examples[1]); + $this->assertEquals(32, strlen($key1['key'])); + $this->assertEquals('aes-cbc', $key1['suite']); + $this->assertEquals('0ao5eC7C/rwwk2qii4oLd6eG3KJq8ZDX2K9zWbvaLdo=', base64_encode($key1['key'])); + + $key2 = $registry->parseKey($examples[2]); + $this->assertEquals(32, strlen($key2['key'])); + $this->assertEquals('aes-ctr', $key2['suite']); + $this->assertEquals('0ao5eC7C/rwwk2qii4oLd6eG3KJq8ZDX2K9zWbvaLdo=', base64_encode($key2['key'])); + + $key3 = $registry->parseKey($examples[3]); + $this->assertEquals(32, strlen($key3['key'])); + $this->assertEquals('aes-cbc-hs', $key3['suite']); + $this->assertEquals('0ao5eC7C/rwwk2qii4oLd6eG3KJq8ZDX2K9zWbvaLdo=', base64_encode($key3['key'])); + } + + public function testRegisterAndFindKeys() { + /** @var CryptoRegistry $registry */ + $registry = \Civi::service('crypto.registry'); + + $key = $registry->findKey('asdf-key-0'); + $this->assertEquals(32, strlen($key['key'])); + $this->assertEquals('aes-cbc', $key['suite']); + + $key = $registry->findKey('asdf-key-1'); + $this->assertEquals(32, strlen($key['key'])); + $this->assertEquals('aes-cbc', $key['suite']); + + $key = $registry->findKey('asdf-key-2'); + $this->assertEquals(32, strlen($key['key'])); + $this->assertEquals('aes-ctr', $key['suite']); + + $key = $registry->findKey('asdf-key-3'); + $this->assertEquals(32, strlen($key['key'])); + $this->assertEquals('aes-cbc-hs', $key['suite']); + + $key = $registry->findKey('UNIT-TEST'); + $this->assertEquals(32, strlen($key['key'])); + $this->assertEquals('asdf-key-1', $key['id']); + } + + public function testValidKeyId() { + $valids = ['abc', 'a.b-c_d+e/', 'f\\g:h;i=']; + $invalids = [chr(0), chr(1), chr(1) . 'abc', 'a b', "ab\n", "ab\nc", "\r", "\n"]; + + /** @var CryptoRegistry $registry */ + $registry = \Civi::service('crypto.registry'); + + foreach ($valids as $valid) { + $this->assertEquals(TRUE, $registry->isValidKeyId($valid), "Key ID \"$valid\" should be valid"); + } + + foreach ($invalids as $invalid) { + $this->assertEquals(FALSE, $registry->isValidKeyId($invalid), "Key ID \"$invalid\" should be invalid"); + } + } + + public function testAddBadKeyId() { + /** @var CryptoRegistry $registry */ + $registry = \Civi::service('crypto.registry'); + + try { + $registry->addSymmetricKey([ + 'key' => 'abcd', + 'id' => "foo\n", + ]); + $this->fail("Expected crypto exception"); + } + catch (CryptoException $e) { + $this->assertRegExp(';Malformed key ID;', $e->getMessage()); + } + } + +} diff --git a/tests/phpunit/Civi/Crypto/CryptoTestTrait.php b/tests/phpunit/Civi/Crypto/CryptoTestTrait.php new file mode 100644 index 000000000000..e20eefdc01de --- /dev/null +++ b/tests/phpunit/Civi/Crypto/CryptoTestTrait.php @@ -0,0 +1,63 @@ +getKeys()); + + $examples = self::getExampleKeys(); + $key = $registry->addSymmetricKey($registry->parseKey($examples[0]) + [ + 'tags' => ['UNIT-TEST'], + 'weight' => 10, + 'id' => 'asdf-key-0', + ]); + $this->assertEquals(10, $key['weight']); + + $key = $registry->addSymmetricKey($registry->parseKey($examples[1]) + [ + 'tags' => ['UNIT-TEST'], + 'weight' => -10, + 'id' => 'asdf-key-1', + ]); + $this->assertEquals(-10, $key['weight']); + + $key = $registry->addSymmetricKey($registry->parseKey($examples[2]) + [ + 'tags' => ['UNIT-TEST'], + 'id' => 'asdf-key-2', + ]); + $this->assertEquals(0, $key['weight']); + + $key = $registry->addSymmetricKey($registry->parseKey($examples[3]) + [ + 'tags' => ['UNIT-TEST'], + 'id' => 'asdf-key-3', + ]); + $this->assertEquals(0, $key['weight']); + + $this->assertEquals(4, count($examples)); + $this->assertEquals(4 + $origCount, count($registry->getKeys())); + } + +} From 3fef2e2135ae4f05d8d6d506f97fa150a338c44f Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Wed, 16 Dec 2020 13:27:33 -0800 Subject: [PATCH 23/63] (dev/core#2258) PhpseclibCipherSuite - Add default crypto provider --- Civi/Core/Container.php | 1 + Civi/Crypto/PhpseclibCipherSuite.php | 213 +++++++++++++++++++++++++++ 2 files changed, 214 insertions(+) create mode 100644 Civi/Crypto/PhpseclibCipherSuite.php diff --git a/Civi/Core/Container.php b/Civi/Core/Container.php index 404405adadb9..6c8ac4ffbc25 100644 --- a/Civi/Core/Container.php +++ b/Civi/Core/Container.php @@ -511,6 +511,7 @@ public static function createCacheConfig() { */ public static function createCryptoRegistry() { $crypto = new \Civi\Crypto\CryptoRegistry(); + $crypto->addCipherSuite(new \Civi\Crypto\PhpseclibCipherSuite()); $crypto->addPlainText(['tags' => ['CRED']]); if (defined('CIVICRM_CRED_KEYS')) { diff --git a/Civi/Crypto/PhpseclibCipherSuite.php b/Civi/Crypto/PhpseclibCipherSuite.php new file mode 100644 index 000000000000..61683307540b --- /dev/null +++ b/Civi/Crypto/PhpseclibCipherSuite.php @@ -0,0 +1,213 @@ +ciphers = []; + if (class_exists('\phpseclib\Crypt\AES')) { + // phpseclib v2 + $this->ciphers['aes-cbc'] = new \phpseclib\Crypt\AES(\phpseclib\Crypt\AES::MODE_CBC); + $this->ciphers['aes-cbc']->setKeyLength(256); + $this->ciphers['aes-ctr'] = new \phpseclib\Crypt\AES(\phpseclib\Crypt\AES::MODE_CTR); + $this->ciphers['aes-ctr']->setKeyLength(256); + } + elseif (class_exists('Crypt_AES')) { + // phpseclib v1 + $this->ciphers['aes-cbc'] = new \Crypt_AES(\Crypt_AES::MODE_CBC); + $this->ciphers['aes-cbc']->setKeyLength(256); + $this->ciphers['aes-ctr'] = new \Crypt_AES(\Crypt_AES::MODE_CTR); + $this->ciphers['aes-ctr']->setKeyLength(256); + } + else { + throw new CryptoException("Failed to find phpseclib"); + } + } + + /** + * @inheritdoc + */ + public function getSuites(): array { + return ['aes-cbc', 'aes-ctr', 'aes-cbc-hs', 'aes-ctr-hs']; + } + + /** + * @inheritdoc + */ + public function encrypt(string $plainText, array $key): string { + switch ($key['suite']) { + case 'aes-cbc-hs': + case 'aes-ctr-hs': + return $this->encryptThenSign($plainText, substr($key['suite'], 0, -3), 'sha256', $key['key']); + + case 'aes-cbc': + case 'aes-ctr': + return $this->encryptOnly($plainText, $key['suite'], $key['key']); + } + } + + /** + * @inheritdoc + */ + public function decrypt(string $cipherText, array $key): string { + switch ($key['suite']) { + case 'aes-cbc-hs': + case 'aes-ctr-hs': + return $this->authenticateThenDecrypt($cipherText, substr($key['suite'], 0, -3), 'sha256', $key['key']); + + case 'aes-cbc': + case 'aes-ctr': + return $this->decryptOnly($cipherText, $key['suite'], $key['key']); + } + } + + /** + * Given an master key, derive a pair of encryption+authentication keys. + * + * @param string $masterKey + * @return array + */ + protected function createEncAuthKeys($masterKey) { + return [ + hash_hmac('sha256', 'enc', $masterKey, TRUE), + hash_hmac('sha256', 'auth', $masterKey, TRUE), + ]; + } + + protected function encryptOnly($plainText, $suite, $key) { + $cipher = $this->createCipher($suite, $key); + $blockBytes = $cipher->getBlockLength() >> 3; + $iv = random_bytes($blockBytes); + $cipher->setIV($iv); + return $iv . $cipher->encrypt($plainText); + } + + protected function decryptOnly(string $cipherText, $suite, $key) { + $cipher = $this->createCipher($suite, $key); + $blockBytes = $cipher->getBlockLength() >> 3; + $iv = substr($cipherText, 0, $blockBytes); + $cipher->setIV($iv); + return $cipher->decrypt(substr($cipherText, $blockBytes)); + } + + /** + * @param string $plainText + * @param string $suite + * The encryption algorithms + * Ex: aes-cbc, aes-ctr + * @param string $digest + * The authentication algorithm + * Ex: sha256 + * @param string $masterKey + * Binary representation of the key + * + * @return string + * The concatenation of IV, ciphertext, signature + */ + protected function encryptThenSign($plainText, $suite, $digest, $masterKey) { + list ($encKey, $authKey) = $this->createEncAuthKeys($masterKey); + $cipher = $this->createCipher($suite, $encKey); + $blockBytes = $cipher->getBlockLength() >> 3; + $iv = random_bytes($blockBytes); + $cipher->setIV($iv); + $ivText = $iv . $cipher->encrypt($plainText); + $sig = hash_hmac($digest, $ivText, $authKey, TRUE); + $this->assertLen($this->getDigestBytes($digest), $sig); + return $ivText . $sig; + } + + /** + * @param string $cipherText + * Combined ciphertext (IV + encrypted text + signature) + * @param string $suite + * The encryption algorithms + * Ex: aes-cbc, aes-ctr + * @param string $digest + * The authentication algorithm + * Ex: sha256 + * @param string $masterKey + * Binary representation of the key + * + * @return string + * Decrypted text + * @throws CryptoException + * Throws an exception if authentication fails. + */ + protected function authenticateThenDecrypt($cipherText, $suite, $digest, $masterKey) { + list ($encKey, $authKey) = $this->createEncAuthKeys($masterKey); + $cipher = $this->createCipher($suite, $encKey); + $blockBytes = $cipher->getBlockLength() >> 3; + $digestBytes = $this->getDigestBytes($digest); + $sigExpect = substr($cipherText, -1 * $digestBytes); + $sigActual = hash_hmac($digest, substr($cipherText, 0, -1 * $digestBytes), $authKey, TRUE); + if (!hash_equals($sigActual, $sigExpect)) { + throw new CryptoException("Failed to decrypt token. Invalid digest."); + } + $cipher->setIV(substr($cipherText, 0, $blockBytes)); + return $cipher->decrypt(substr($cipherText, $blockBytes, -1 * $digestBytes)); + } + + /** + * @param $suite + * @param $key + * @return \phpseclib\Crypt\Base|\Crypt_Base + */ + protected function createCipher($suite, $key) { + if (!isset($this->ciphers[$suite])) { + throw new \RuntimeException("Cipher suite does not support " . $suite); + } + + $cipher = clone $this->ciphers[$suite]; + $this->assertLen($cipher->getKeyLength() >> 3, $key); + $cipher->setKey($key); + return $cipher; + } + + protected function getDigestBytes($digest) { + if ($digest === 'sha256') { + return 32; + } + throw new \RuntimeException('Unrecognized digest'); + } + + private function assertLen($bytes, $value) { + if ($bytes != strlen($value)) { + throw new \InvalidArgumentException("Malformed AES key"); + } + } + +} From 8d3452c40914685bfcbdd48ce9e471415e4d1d51 Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Wed, 16 Dec 2020 15:19:09 -0800 Subject: [PATCH 24/63] (dev/core#2258) CryptoToken - Add service for processing stored ciphertext --- Civi/Core/Container.php | 5 + Civi/Crypto/CryptoToken.php | 140 ++++++++++++++++++ tests/phpunit/Civi/Crypto/CryptoTokenTest.php | 69 +++++++++ 3 files changed, 214 insertions(+) create mode 100644 Civi/Crypto/CryptoToken.php create mode 100644 tests/phpunit/Civi/Crypto/CryptoTokenTest.php diff --git a/Civi/Core/Container.php b/Civi/Core/Container.php index 6c8ac4ffbc25..fb3f0b9d7912 100644 --- a/Civi/Core/Container.php +++ b/Civi/Core/Container.php @@ -221,6 +221,11 @@ public function createContainer() { $container->setDefinition('crypto.registry', new Definition('Civi\Crypto\CryptoService')) ->setFactory(__CLASS__ . '::createCryptoRegistry')->setPublic(TRUE); + $container->setDefinition('crypto.token', new Definition( + 'Civi\Crypto\CryptoToken', + [new Reference('crypto.registry')] + ))->setPublic(TRUE); + if (empty(\Civi::$statics[__CLASS__]['boot'])) { throw new \RuntimeException('Cannot initialize container. Boot services are undefined.'); } diff --git a/Civi/Crypto/CryptoToken.php b/Civi/Crypto/CryptoToken.php new file mode 100644 index 000000000000..a14ef8c54802 --- /dev/null +++ b/Civi/Crypto/CryptoToken.php @@ -0,0 +1,140 @@ +encrypt('my-mail-password, 'KEY_ID_OR_TAG'); + * $decrypted = Civi::service('crypto.token')->decrypt($encrypted, '*'); + * + * FORMAT: An encoded token may be in either of these formats: + * + * - Plain text: Any string which does not begin with chr(2) + * - Encrypted text: A string in the format: + * TOKEN := DLM + VERSION + DLM + KEY_ID + DLM + CIPHERTEXT + * DLM := ASCII CHAR #2 + * VERSION := String, 4-digit, alphanumeric (as in "CTK0") + * KEY_ID := String, alphanumeric and symbols "_-.,:;=+/\" + * + * @package Civi\Crypto + */ +class CryptoToken { + + const VERSION_1 = 'CTK0'; + + /** + * @var CryptoRegistry + */ + protected $registry; + + protected $delim; + + /** + * CryptoToken constructor. + * @param \Civi\Crypto\CryptoRegistry $registry + */ + public function __construct(\Civi\Crypto\CryptoRegistry $registry) { + $this->delim = chr(2); + $this->registry = $registry; + } + + /** + * Determine if a string looks like plain-text. + * + * @param string $plainText + * @return bool + */ + public function isPlainText($plainText) { + return is_string($plainText) && ($plainText === '' || $plainText{0} !== $this->delim); + } + + /** + * Create an encrypted token (given the plaintext). + * + * @param string $plainText + * The secret value to encode (e.g. plain-text password). + * @param string|string[] $keyIdOrTag + * List of key IDs or key tags to check. First available match wins. + * @return string + * A token + * @throws \Civi\Crypto\Exception\CryptoException + */ + public function encrypt($plainText, $keyIdOrTag) { + $key = $this->registry->findKey($keyIdOrTag); + if ($key['suite'] === 'plain') { + if (!$this->isPlainText($plainText)) { + throw new CryptoException("Cannot use plaintext encoding for data with reserved delimiter."); + } + return $plainText; + } + + /** @var \Civi\Crypto\CipherSuiteInterface $cipherSuite */ + $cipherSuite = $this->registry->findSuite($key['suite']); + $cipherText = $cipherSuite->encrypt($plainText, $key); + return $this->delim . self::VERSION_1 . $this->delim . $key['id'] . $this->delim . base64_encode($cipherText); + } + + /** + * Get the plaintext (given an encrypted token). + * + * @param string $token + * @param string|string[] $keyIdOrTag + * Whitelist of acceptable keys. Wildcard '*' will allow it to use + * any/all available means to decode the token. + * @return string + * @throws \Civi\Crypto\Exception\CryptoException + */ + public function decrypt($token, $keyIdOrTag = '*') { + $keyIdOrTag = (array) $keyIdOrTag; + + if ($this->isPlainText($token)) { + if (in_array('*', $keyIdOrTag) || in_array('plain', $keyIdOrTag)) { + return $token; + } + else { + throw new CryptoException("Cannot decrypt token. Unexpected key: plain"); + } + } + + $parts = explode($this->delim, $token); + if ($parts[1] !== self::VERSION_1) { + throw new CryptoException("Unrecognized encoding"); + } + $keyId = $parts[2]; + $cipherText = base64_decode($parts[3]); + + $key = $this->registry->findKey($keyId); + if (!in_array('*', $keyIdOrTag) && !in_array($keyId, $keyIdOrTag) && empty(array_intersect($keyIdOrTag, $key['tags']))) { + throw new CryptoException("Cannot decrypt token. Unexpected key: $keyId"); + } + + /** @var \Civi\Crypto\CipherSuiteInterface $cipherSuite */ + $cipherSuite = $this->registry->findSuite($key['suite']); + $plainText = $cipherSuite->decrypt($cipherText, $key); + return $plainText; + } + +} diff --git a/tests/phpunit/Civi/Crypto/CryptoTokenTest.php b/tests/phpunit/Civi/Crypto/CryptoTokenTest.php new file mode 100644 index 000000000000..961166a89a01 --- /dev/null +++ b/tests/phpunit/Civi/Crypto/CryptoTokenTest.php @@ -0,0 +1,69 @@ +setHook('civicrm_crypto', [$this, 'registerExampleKeys']); + } + + public function testIsPlainText() { + $token = \Civi::service('crypto.token'); + + $this->assertFalse($token->isPlainText(chr(2))); + $this->assertFalse($token->isPlainText(chr(2) . 'asdf')); + + $this->assertTrue($token->isPlainText(\CRM_Utils_Array::implodePadded(['a', 'b', 'c']))); + $this->assertTrue($token->isPlainText("")); + $this->assertTrue($token->isPlainText("\r")); + $this->assertTrue($token->isPlainText("\n")); + } + + public function getExampleTokens() { + return [ + // [ 'Plain text', 'Encryption Key ID', 'expectTokenRegex', 'expectTokenLen', 'expectPlain' ] + ['hello world. can you see me', 'plain', '/^hello world. can you see me/', 27, TRUE], + ['hello world. i am secret.', 'UNIT-TEST', '/^.CTK0.asdf-key-1./', 81, FALSE], + ['hello world. we b secret.', 'asdf-key-0', '/^.CTK0.asdf-key-0./', 81, FALSE], + ['hello world. u ur secret.', 'asdf-key-1', '/^.CTK0.asdf-key-1./', 81, FALSE], + ['hello world. he z secret.', 'asdf-key-2', '/^.CTK0.asdf-key-2./', 73, FALSE], + ['hello world. whos secret.', 'asdf-key-3', '/^.CTK0.asdf-key-3./', 125, FALSE], + ]; + } + + /** + * @param string $inputText + * @param string $inputKeyIdOrTag + * @param string $expectTokenRegex + * @param int $expectTokenLen + * @param bool $expectPlain + * + * @dataProvider getExampleTokens + */ + public function testRoundtrip($inputText, $inputKeyIdOrTag, $expectTokenRegex, $expectTokenLen, $expectPlain) { + $token = \Civi::service('crypto.token')->encrypt($inputText, $inputKeyIdOrTag); + $this->assertRegExp($expectTokenRegex, $token); + $this->assertEquals($expectTokenLen, strlen($token)); + $this->assertEquals($expectPlain, \Civi::service('crypto.token')->isPlainText($token)); + + $actualText = \Civi::service('crypto.token')->decrypt($token); + $this->assertEquals($inputText, $actualText); + } + +} From 7c5110c362e57b9c424829070f8e13274bbe0cfd Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Wed, 16 Dec 2020 15:29:43 -0800 Subject: [PATCH 25/63] (dev/core#2258) CryptoToken - Don't build registry if we only read plaintext Example: You have a basic site that has not enabled encrypted fields, but the integration with settings/APIs causes one to use CryptoToken::decrypt() --- Civi/Core/Container.php | 6 ++--- Civi/Crypto/CryptoToken.php | 23 +++++++++---------- tests/phpunit/Civi/Crypto/CryptoTokenTest.php | 8 +++++++ 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/Civi/Core/Container.php b/Civi/Core/Container.php index fb3f0b9d7912..67f5319585a6 100644 --- a/Civi/Core/Container.php +++ b/Civi/Core/Container.php @@ -221,10 +221,8 @@ public function createContainer() { $container->setDefinition('crypto.registry', new Definition('Civi\Crypto\CryptoService')) ->setFactory(__CLASS__ . '::createCryptoRegistry')->setPublic(TRUE); - $container->setDefinition('crypto.token', new Definition( - 'Civi\Crypto\CryptoToken', - [new Reference('crypto.registry')] - ))->setPublic(TRUE); + $container->setDefinition('crypto.token', new Definition('Civi\Crypto\CryptoToken', [])) + ->setPublic(TRUE); if (empty(\Civi::$statics[__CLASS__]['boot'])) { throw new \RuntimeException('Cannot initialize container. Boot services are undefined.'); diff --git a/Civi/Crypto/CryptoToken.php b/Civi/Crypto/CryptoToken.php index a14ef8c54802..93fe284fc894 100644 --- a/Civi/Crypto/CryptoToken.php +++ b/Civi/Crypto/CryptoToken.php @@ -45,20 +45,13 @@ class CryptoToken { const VERSION_1 = 'CTK0'; - /** - * @var CryptoRegistry - */ - protected $registry; - protected $delim; /** * CryptoToken constructor. - * @param \Civi\Crypto\CryptoRegistry $registry */ - public function __construct(\Civi\Crypto\CryptoRegistry $registry) { + public function __construct() { $this->delim = chr(2); - $this->registry = $registry; } /** @@ -83,7 +76,10 @@ public function isPlainText($plainText) { * @throws \Civi\Crypto\Exception\CryptoException */ public function encrypt($plainText, $keyIdOrTag) { - $key = $this->registry->findKey($keyIdOrTag); + /** @var CryptoRegistry $registry */ + $registry = \Civi::service('crypto.registry'); + + $key = $registry->findKey($keyIdOrTag); if ($key['suite'] === 'plain') { if (!$this->isPlainText($plainText)) { throw new CryptoException("Cannot use plaintext encoding for data with reserved delimiter."); @@ -92,7 +88,7 @@ public function encrypt($plainText, $keyIdOrTag) { } /** @var \Civi\Crypto\CipherSuiteInterface $cipherSuite */ - $cipherSuite = $this->registry->findSuite($key['suite']); + $cipherSuite = $registry->findSuite($key['suite']); $cipherText = $cipherSuite->encrypt($plainText, $key); return $this->delim . self::VERSION_1 . $this->delim . $key['id'] . $this->delim . base64_encode($cipherText); } @@ -119,6 +115,9 @@ public function decrypt($token, $keyIdOrTag = '*') { } } + /** @var CryptoRegistry $registry */ + $registry = \Civi::service('crypto.registry'); + $parts = explode($this->delim, $token); if ($parts[1] !== self::VERSION_1) { throw new CryptoException("Unrecognized encoding"); @@ -126,13 +125,13 @@ public function decrypt($token, $keyIdOrTag = '*') { $keyId = $parts[2]; $cipherText = base64_decode($parts[3]); - $key = $this->registry->findKey($keyId); + $key = $registry->findKey($keyId); if (!in_array('*', $keyIdOrTag) && !in_array($keyId, $keyIdOrTag) && empty(array_intersect($keyIdOrTag, $key['tags']))) { throw new CryptoException("Cannot decrypt token. Unexpected key: $keyId"); } /** @var \Civi\Crypto\CipherSuiteInterface $cipherSuite */ - $cipherSuite = $this->registry->findSuite($key['suite']); + $cipherSuite = $registry->findSuite($key['suite']); $plainText = $cipherSuite->decrypt($cipherText, $key); return $plainText; } diff --git a/tests/phpunit/Civi/Crypto/CryptoTokenTest.php b/tests/phpunit/Civi/Crypto/CryptoTokenTest.php index 961166a89a01..64d3e310a98a 100644 --- a/tests/phpunit/Civi/Crypto/CryptoTokenTest.php +++ b/tests/phpunit/Civi/Crypto/CryptoTokenTest.php @@ -66,4 +66,12 @@ public function testRoundtrip($inputText, $inputKeyIdOrTag, $expectTokenRegex, $ $this->assertEquals($inputText, $actualText); } + public function testReadPlainTextWithoutRegistry() { + // This is performance optimization - don't initialize crypto.registry unless + // you actually need it. + $this->assertFalse(\Civi::container()->initialized('crypto.registry')); + $this->assertEquals("Hello world", \Civi::service('crypto.token')->decrypt("Hello world")); + $this->assertFalse(\Civi::container()->initialized('crypto.registry')); + } + } From 23fa01181f70f92fb3ac319cfb9e8644f15dfb4d Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Wed, 16 Dec 2020 15:41:48 -0800 Subject: [PATCH 26/63] (dev/core#2258) CryptoRegistry - Don't parse factory code unless it's being used This moves the factory function from `Container.php` (which is loaded on all page-views on all configurations) to `CryptoRegistry.php` (which is only loaded if the site actually used encrypted fields). --- Civi/Core/Container.php | 45 +--------------------------------- Civi/Crypto/CryptoRegistry.php | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/Civi/Core/Container.php b/Civi/Core/Container.php index 67f5319585a6..1eec43c27ac2 100644 --- a/Civi/Core/Container.php +++ b/Civi/Core/Container.php @@ -219,7 +219,7 @@ public function createContainer() { ->setFactory('CRM_Utils_Mail::createMailer')->setPublic(TRUE); $container->setDefinition('crypto.registry', new Definition('Civi\Crypto\CryptoService')) - ->setFactory(__CLASS__ . '::createCryptoRegistry')->setPublic(TRUE); + ->setFactory('Civi\Crypto\CryptoRegistry::createDefaultRegistry')->setPublic(TRUE); $container->setDefinition('crypto.token', new Definition('Civi\Crypto\CryptoToken', [])) ->setPublic(TRUE); @@ -505,49 +505,6 @@ public static function createCacheConfig() { return new \ArrayObject($settings); } - /** - * Initialize the cryptogrpahic registry. It tracks available ciphers and keys. - * - * @return \Civi\Crypto\CryptoRegistry - * @throws \CRM_Core_Exception - * @throws \Civi\Crypto\Exception\CryptoException - */ - public static function createCryptoRegistry() { - $crypto = new \Civi\Crypto\CryptoRegistry(); - $crypto->addCipherSuite(new \Civi\Crypto\PhpseclibCipherSuite()); - - $crypto->addPlainText(['tags' => ['CRED']]); - if (defined('CIVICRM_CRED_KEYS')) { - foreach (explode(' ', CIVICRM_CRED_KEYS) as $n => $keyExpr) { - $crypto->addSymmetricKey($crypto->parseKey($keyExpr) + [ - 'tags' => ['CRED'], - 'weight' => $n, - ]); - } - } - if (defined('CIVICRM_SITE_KEY')) { - // Recent upgrades may not have CIVICRM_CRED_KEYS. Transitional support - the CIVICRM_SITE_KEY is last-priority option for credentials. - $crypto->addSymmetricKey([ - 'key' => hash_hkdf('sha256', CIVICRM_SITE_KEY), - 'suite' => 'aes-cbc', - 'tags' => ['CRED'], - 'weight' => 30000, - ]); - } - //if (isset($_COOKIE['CIVICRM_FORM_KEY'])) { - // $crypto->addSymmetricKey([ - // 'key' => base64_decode($_COOKIE['CIVICRM_FORM_KEY']), - // 'suite' => 'aes-cbc', - // 'tag' => ['FORM'], - // ]); - // // else: somewhere in CRM_Core_Form, we may need to initialize CIVICRM_FORM_KEY - //} - - // Allow plugins to add/replace any keys and ciphers. - \CRM_Utils_Hook::crypto($crypto); - return $crypto; - } - /** * Get a list of boot services. * diff --git a/Civi/Crypto/CryptoRegistry.php b/Civi/Crypto/CryptoRegistry.php index 5e660b4a321b..7fc2150ad9da 100644 --- a/Civi/Crypto/CryptoRegistry.php +++ b/Civi/Crypto/CryptoRegistry.php @@ -54,6 +54,49 @@ class CryptoRegistry { protected $cipherSuites = []; + /** + * Initialize a default instance of the registry. + * + * @return \Civi\Crypto\CryptoRegistry + * @throws \CRM_Core_Exception + * @throws \Civi\Crypto\Exception\CryptoException + */ + public static function createDefaultRegistry() { + $registry = new static(); + $registry->addCipherSuite(new \Civi\Crypto\PhpseclibCipherSuite()); + + $registry->addPlainText(['tags' => ['CRED']]); + if (defined('CIVICRM_CRED_KEYS')) { + foreach (explode(' ', CIVICRM_CRED_KEYS) as $n => $keyExpr) { + $registry->addSymmetricKey($registry->parseKey($keyExpr) + [ + 'tags' => ['CRED'], + 'weight' => $n, + ]); + } + } + if (defined('CIVICRM_SITE_KEY')) { + // Recent upgrades may not have CIVICRM_CRED_KEYS. Transitional support - the CIVICRM_SITE_KEY is last-priority option for credentials. + $registry->addSymmetricKey([ + 'key' => hash_hkdf('sha256', CIVICRM_SITE_KEY), + 'suite' => 'aes-cbc', + 'tags' => ['CRED'], + 'weight' => 30000, + ]); + } + //if (isset($_COOKIE['CIVICRM_FORM_KEY'])) { + // $crypto->addSymmetricKey([ + // 'key' => base64_decode($_COOKIE['CIVICRM_FORM_KEY']), + // 'suite' => 'aes-cbc', + // 'tag' => ['FORM'], + // ]); + // // else: somewhere in CRM_Core_Form, we may need to initialize CIVICRM_FORM_KEY + //} + + // Allow plugins to add/replace any keys and ciphers. + \CRM_Utils_Hook::crypto($registry); + return $registry; + } + public function __construct() { $this->cipherSuites['plain'] = TRUE; $this->keys['plain'] = [ From 27fe36e71e01dacd1d7b373294795876e3c49c3d Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Wed, 16 Dec 2020 16:26:15 -0800 Subject: [PATCH 27/63] (dev/core#2258) CryptoToken - More defense against malformed input --- Civi/Crypto/CryptoToken.php | 6 ++--- tests/phpunit/Civi/Crypto/CryptoTokenTest.php | 25 +++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/Civi/Crypto/CryptoToken.php b/Civi/Crypto/CryptoToken.php index 93fe284fc894..e45f962b9fa5 100644 --- a/Civi/Crypto/CryptoToken.php +++ b/Civi/Crypto/CryptoToken.php @@ -118,9 +118,9 @@ public function decrypt($token, $keyIdOrTag = '*') { /** @var CryptoRegistry $registry */ $registry = \Civi::service('crypto.registry'); - $parts = explode($this->delim, $token); - if ($parts[1] !== self::VERSION_1) { - throw new CryptoException("Unrecognized encoding"); + $parts = explode($this->delim, $token, 4); + if (count($parts) !== 4 || $parts[1] !== self::VERSION_1) { + throw new CryptoException("Cannot decrypt token. Invalid format."); } $keyId = $parts[2]; $cipherText = base64_decode($parts[3]); diff --git a/tests/phpunit/Civi/Crypto/CryptoTokenTest.php b/tests/phpunit/Civi/Crypto/CryptoTokenTest.php index 64d3e310a98a..fa949ca5cdda 100644 --- a/tests/phpunit/Civi/Crypto/CryptoTokenTest.php +++ b/tests/phpunit/Civi/Crypto/CryptoTokenTest.php @@ -11,6 +11,8 @@ namespace Civi\Crypto; +use Civi\Crypto\Exception\CryptoException; + /** * Test major use-cases of the 'crypto.token' service. */ @@ -35,6 +37,29 @@ public function testIsPlainText() { $this->assertTrue($token->isPlainText("\n")); } + public function testDecryptInvalid() { + $cryptoToken = \Civi::service('crypto.token'); + try { + $cryptoToken->decrypt(chr(2) . 'CTK0' . chr(2)); + $this->fail("Expected CryptoException"); + } + catch (CryptoException $e) { + $this->assertRegExp(';Cannot decrypt token. Invalid format.;', $e->getMessage()); + } + + $goodExample = $cryptoToken->encrypt('mess with me', 'UNIT-TEST'); + $this->assertEquals('mess with me', $cryptoToken->decrypt($goodExample)); + + try { + $badExample = preg_replace(';CTK0;', 'ctk9', $goodExample); + $cryptoToken->decrypt($badExample); + $this->fail("Expected CryptoException"); + } + catch (CryptoException $e) { + $this->assertRegExp(';Cannot decrypt token. Invalid format.;', $e->getMessage()); + } + } + public function getExampleTokens() { return [ // [ 'Plain text', 'Encryption Key ID', 'expectTokenRegex', 'expectTokenLen', 'expectPlain' ] From f702c7f5d4cd9ad43da1f747317f9bc004b2e989 Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Wed, 16 Dec 2020 16:53:47 -0800 Subject: [PATCH 28/63] (dev/core#2258) CryptoRegistry - Remove fallback option based on CIVICRM_SITE_KEY. This was included under the expectation it might make a nicer upgrade. But I don't think it buys a whole lot: 1. You run the upgrader. The SMTP password is converted from rj256-ecb-sitekey to aes-cbc-sitekey. All other credentials are left unencrypted. Afterward, you set CIVICRM_CRED_KEY and run the rekey. 2. You run upgrader. The SMTP password is decrypted. All other credentials are left unencrypted. Afterward, you set CIVICRM_CRED_KEY and run the rekey. Additionally, I think there's a question of risk-management when we get to encrypting more things in the Setting and API layers. If we go with path 2, then we can ramp-up adoption progressively, e.g. * Release 1: Add support as non-default option * Release 2: Enable by default on new builds * Release 3: Display alert to existing sites that don't have encryption keys --- Civi/Crypto/CryptoRegistry.php | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/Civi/Crypto/CryptoRegistry.php b/Civi/Crypto/CryptoRegistry.php index 7fc2150ad9da..0cca04fe8912 100644 --- a/Civi/Crypto/CryptoRegistry.php +++ b/Civi/Crypto/CryptoRegistry.php @@ -74,15 +74,7 @@ public static function createDefaultRegistry() { ]); } } - if (defined('CIVICRM_SITE_KEY')) { - // Recent upgrades may not have CIVICRM_CRED_KEYS. Transitional support - the CIVICRM_SITE_KEY is last-priority option for credentials. - $registry->addSymmetricKey([ - 'key' => hash_hkdf('sha256', CIVICRM_SITE_KEY), - 'suite' => 'aes-cbc', - 'tags' => ['CRED'], - 'weight' => 30000, - ]); - } + //if (isset($_COOKIE['CIVICRM_FORM_KEY'])) { // $crypto->addSymmetricKey([ // 'key' => base64_decode($_COOKIE['CIVICRM_FORM_KEY']), From 4228925bae872ddb4e5f268c1516c8e820bdd580 Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Fri, 18 Dec 2020 13:14:17 -0800 Subject: [PATCH 29/63] (NFC) CRM_Utils_Hook - Tweak description of crypto() --- CRM/Utils/Hook.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CRM/Utils/Hook.php b/CRM/Utils/Hook.php index 8ec1d7235da5..920cdf74e145 100644 --- a/CRM/Utils/Hook.php +++ b/CRM/Utils/Hook.php @@ -1760,9 +1760,12 @@ public static function alterTemplateFile($formName, &$form, $context, &$tplName) } /** - * Initialize the cryptographic service. + * Register cryptographic resources, such as keys and cipher-suites. * - * This may be used to register additional keys or cipher-suites. + * Ex: $crypto->addSymmetricKey([ + * 'key' => hash_hkdf('sha256', 'abcd1234'), + * 'suite' => 'aes-cbc-hs', + * ]); * * @param \Civi\Crypto\CryptoRegistry $crypto * From 5b394c8f531cb30eccf5507bf7c08997603bf217 Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Fri, 18 Dec 2020 20:47:31 -0800 Subject: [PATCH 30/63] (dev/core#2258) CryptoToken - Change notation This is pre-merge change to the notation in the token. Two things: * Use only one control character instead of multiple. * Use URL-style key-value notation. It should be easier to skim and to tweak. --- Civi/Crypto/CryptoToken.php | 41 +++++++++++++------ tests/phpunit/Civi/Crypto/CryptoTokenTest.php | 13 +++--- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/Civi/Crypto/CryptoToken.php b/Civi/Crypto/CryptoToken.php index e45f962b9fa5..038f405ce2d2 100644 --- a/Civi/Crypto/CryptoToken.php +++ b/Civi/Crypto/CryptoToken.php @@ -34,17 +34,25 @@ * * - Plain text: Any string which does not begin with chr(2) * - Encrypted text: A string in the format: - * TOKEN := DLM + VERSION + DLM + KEY_ID + DLM + CIPHERTEXT - * DLM := ASCII CHAR #2 - * VERSION := String, 4-digit, alphanumeric (as in "CTK0") - * KEY_ID := String, alphanumeric and symbols "_-.,:;=+/\" + * TOKEN := DLM + FMT + QUERY + * DLM := ASCII char #2 + * FMT := String, 4-digit, alphanumeric (as in "CTK?") + * QUERY := String, URL-encoded key-value pairs, + * "k", the key ID (alphanumeric and symbols "_-.,:;=+/\") + * "t", the text (base64-encoded ciphertext) * * @package Civi\Crypto */ class CryptoToken { - const VERSION_1 = 'CTK0'; + /** + * Format identification code + */ + const FMT_QUERY = 'CTK?'; + /** + * @var string + */ protected $delim; /** @@ -90,7 +98,11 @@ public function encrypt($plainText, $keyIdOrTag) { /** @var \Civi\Crypto\CipherSuiteInterface $cipherSuite */ $cipherSuite = $registry->findSuite($key['suite']); $cipherText = $cipherSuite->encrypt($plainText, $key); - return $this->delim . self::VERSION_1 . $this->delim . $key['id'] . $this->delim . base64_encode($cipherText); + + return $this->delim . self::FMT_QUERY . \http_build_query([ + 'k' => $key['id'], + 't' => \CRM_Utils_String::base64UrlEncode($cipherText), + ]); } /** @@ -118,16 +130,21 @@ public function decrypt($token, $keyIdOrTag = '*') { /** @var CryptoRegistry $registry */ $registry = \Civi::service('crypto.registry'); - $parts = explode($this->delim, $token, 4); - if (count($parts) !== 4 || $parts[1] !== self::VERSION_1) { - throw new CryptoException("Cannot decrypt token. Invalid format."); + $fmt = substr($token, 1, 4); + switch ($fmt) { + case self::FMT_QUERY: + parse_str(substr($token, 5), $tokenData); + $keyId = $tokenData['k']; + $cipherText = \CRM_Utils_String::base64UrlDecode($tokenData['t']); + break; + + default: + throw new CryptoException("Cannot decrypt token. Invalid format."); } - $keyId = $parts[2]; - $cipherText = base64_decode($parts[3]); $key = $registry->findKey($keyId); if (!in_array('*', $keyIdOrTag) && !in_array($keyId, $keyIdOrTag) && empty(array_intersect($keyIdOrTag, $key['tags']))) { - throw new CryptoException("Cannot decrypt token. Unexpected key: $keyId"); + throw new CryptoException("Cannot decrypt token. Unexpected key: {$keyId}"); } /** @var \Civi\Crypto\CipherSuiteInterface $cipherSuite */ diff --git a/tests/phpunit/Civi/Crypto/CryptoTokenTest.php b/tests/phpunit/Civi/Crypto/CryptoTokenTest.php index fa949ca5cdda..d626698df5db 100644 --- a/tests/phpunit/Civi/Crypto/CryptoTokenTest.php +++ b/tests/phpunit/Civi/Crypto/CryptoTokenTest.php @@ -51,7 +51,8 @@ public function testDecryptInvalid() { $this->assertEquals('mess with me', $cryptoToken->decrypt($goodExample)); try { - $badExample = preg_replace(';CTK0;', 'ctk9', $goodExample); + $badExample = preg_replace(';CTK\?;', 'ctk9', $goodExample); + $this->assertTrue($badExample !== $goodExample); $cryptoToken->decrypt($badExample); $this->fail("Expected CryptoException"); } @@ -64,11 +65,11 @@ public function getExampleTokens() { return [ // [ 'Plain text', 'Encryption Key ID', 'expectTokenRegex', 'expectTokenLen', 'expectPlain' ] ['hello world. can you see me', 'plain', '/^hello world. can you see me/', 27, TRUE], - ['hello world. i am secret.', 'UNIT-TEST', '/^.CTK0.asdf-key-1./', 81, FALSE], - ['hello world. we b secret.', 'asdf-key-0', '/^.CTK0.asdf-key-0./', 81, FALSE], - ['hello world. u ur secret.', 'asdf-key-1', '/^.CTK0.asdf-key-1./', 81, FALSE], - ['hello world. he z secret.', 'asdf-key-2', '/^.CTK0.asdf-key-2./', 73, FALSE], - ['hello world. whos secret.', 'asdf-key-3', '/^.CTK0.asdf-key-3./', 125, FALSE], + ['hello world. i am secret.', 'UNIT-TEST', '/^.CTK\?k=asdf-key-1&/', 84, FALSE], + ['hello world. we b secret.', 'asdf-key-0', '/^.CTK\?k=asdf-key-0&/', 84, FALSE], + ['hello world. u ur secret.', 'asdf-key-1', '/^.CTK\?k=asdf-key-1&/', 84, FALSE], + ['hello world. he z secret.', 'asdf-key-2', '/^.CTK\?k=asdf-key-2&/', 75, FALSE], + ['hello world. whos secret.', 'asdf-key-3', '/^.CTK\?k=asdf-key-3&/', 127, FALSE], ]; } From 945243fb27a87a61129668fc153a3ce5cace3ae1 Mon Sep 17 00:00:00 2001 From: Mikey O'Toole Date: Sat, 19 Dec 2020 19:20:02 +0000 Subject: [PATCH 31/63] Fix issue reporting link to go to the corresponding `core` project. --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index ee36cd697c9d..c66affa92967 100644 --- a/README.md +++ b/README.md @@ -46,5 +46,4 @@ questions and ideas in the [Developer Discussion room](https://chat.civicrm.org/ Installing the latest developmental code requires some [special steps](https://docs.civicrm.org/dev/en/latest/tools/git/). -Report all issues to CiviCRM via GitLab: -https://lab.civicrm.org/ +Report all issues to CiviCRM via GitLab: https://lab.civicrm.org/dev/core From d6694785ca0828ce3f53694a6078be786b383170 Mon Sep 17 00:00:00 2001 From: eileen Date: Mon, 21 Dec 2020 13:21:26 +1300 Subject: [PATCH 32/63] dev/core#2252 remove all handling of strict mode as it has aged out of relevance --- CRM/Core/DAO.php | 10 ---------- CRM/Utils/File.php | 3 --- CRM/Utils/System.php | 12 +++++++++++- distmaker/utils/joomlaxml.php | 2 +- templates/CRM/common/civicrm.settings.php.template | 11 ----------- 5 files changed, 12 insertions(+), 26 deletions(-) diff --git a/CRM/Core/DAO.php b/CRM/Core/DAO.php index 70d248bfb010..0e248c5dda6b 100644 --- a/CRM/Core/DAO.php +++ b/CRM/Core/DAO.php @@ -186,16 +186,6 @@ public static function init($dsn) { } $factory = new CRM_Contact_DAO_Factory(); CRM_Core_DAO::setFactory($factory); - $currentModes = CRM_Utils_SQL::getSqlModes(); - if (CRM_Utils_Constant::value('CIVICRM_MYSQL_STRICT', CRM_Utils_System::isDevelopment())) { - if (CRM_Utils_SQL::supportsFullGroupBy() && !in_array('ONLY_FULL_GROUP_BY', $currentModes) && CRM_Utils_SQL::isGroupByModeInDefault()) { - $currentModes[] = 'ONLY_FULL_GROUP_BY'; - } - if (!in_array('STRICT_TRANS_TABLES', $currentModes)) { - $currentModes = array_merge(['STRICT_TRANS_TABLES'], $currentModes); - } - CRM_Core_DAO::executeQuery("SET SESSION sql_mode = %1", [1 => [implode(',', $currentModes), 'String']]); - } CRM_Core_DAO::executeQuery('SET NAMES utf8mb4'); CRM_Core_DAO::executeQuery('SET @uniqueID = %1', [1 => [CRM_Utils_Request::id(), 'String']]); } diff --git a/CRM/Utils/File.php b/CRM/Utils/File.php index c196b3dd7126..e4fb43b9cc04 100644 --- a/CRM/Utils/File.php +++ b/CRM/Utils/File.php @@ -332,9 +332,6 @@ public static function runSqlQuery($dsn, $queryString, $prefix = NULL, $dieOnErr if (PEAR::isError($db)) { die("Cannot open $dsn: " . $db->getMessage()); } - if (CRM_Utils_Constant::value('CIVICRM_MYSQL_STRICT', CRM_Utils_System::isDevelopment())) { - $db->query('SET SESSION sql_mode = STRICT_TRANS_TABLES'); - } $db->query('SET NAMES utf8mb4'); $transactionId = CRM_Utils_Type::escape(CRM_Utils_Request::id(), 'String'); $db->query('SET @uniqueID = ' . "'$transactionId'"); diff --git a/CRM/Utils/System.php b/CRM/Utils/System.php index 48abe7d3930a..98685df52ba3 100644 --- a/CRM/Utils/System.php +++ b/CRM/Utils/System.php @@ -1829,7 +1829,17 @@ public static function getSiteID() { } /** - * Determine whether this is a developmental system. + * Determine whether this system is deployed using version control. + * + * Normally sites would tune their php error settings to prevent deprecation + * notices appearing on a live site. However, on some systems the user + * does not have control over this setting. Sites with version-controlled + * deployments are unlikely to be in a situation where they cannot alter their + * php error level reporting so we can trust that the are able to set them + * to suppress deprecation / php error level warnings if appropriate but + * in order to phase in deprecation warnings we originally chose not to + * show them on sites who might not be able to set their error_level in + * a way that is appropriate to their site. * * @return bool */ diff --git a/distmaker/utils/joomlaxml.php b/distmaker/utils/joomlaxml.php index fad1ceb9f95d..087555ec64ff 100644 --- a/distmaker/utils/joomlaxml.php +++ b/distmaker/utils/joomlaxml.php @@ -1,5 +1,5 @@ Date: Sat, 19 Dec 2020 19:48:45 -0500 Subject: [PATCH 33/63] Add an explicit alias for sql functions --- ext/search/ang/crmSearchAdmin.module.js | 33 +++++++++++-------- .../crmSearchAdmin.component.js | 5 ++- .../crmSearchFunction.component.js | 7 +++- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/ext/search/ang/crmSearchAdmin.module.js b/ext/search/ang/crmSearchAdmin.module.js index e06b22339db9..7e737d109667 100644 --- a/ext/search/ang/crmSearchAdmin.module.js +++ b/ext/search/ang/crmSearchAdmin.module.js @@ -162,26 +162,31 @@ } } function parseExpr(expr) { - var result = {fn: null, modifier: ''}, - fieldName = expr, - bracketPos = expr.indexOf('('); + if (!expr) { + return; + } + var splitAs = expr.split(' AS '), + info = {fn: null, modifier: ''}, + fieldName = splitAs[0], + bracketPos = splitAs[0].indexOf('('); if (bracketPos >= 0) { - var parsed = expr.substr(bracketPos).match(/[ ]?([A-Z]+[ ]+)?([\w.:]+)/); + var parsed = splitAs[0].substr(bracketPos).match(/[ ]?([A-Z]+[ ]+)?([\w.:]+)/); fieldName = parsed[2]; - result.fn = _.find(CRM.crmSearchAdmin.functions, {name: expr.substring(0, bracketPos)}); - result.modifier = _.trim(parsed[1]); + info.fn = _.find(CRM.crmSearchAdmin.functions, {name: expr.substring(0, bracketPos)}); + info.modifier = _.trim(parsed[1]); } - var fieldAndJoin = expr ? getFieldAndJoin(fieldName, searchEntity) : undefined; - if (fieldAndJoin.field) { + var fieldAndJoin = getFieldAndJoin(fieldName, searchEntity); + if (fieldAndJoin) { var split = fieldName.split(':'), prefixPos = split[0].lastIndexOf(fieldAndJoin.field.name); - result.path = split[0]; - result.prefix = prefixPos > 0 ? result.path.substring(0, prefixPos) : ''; - result.suffix = !split[1] ? '' : ':' + split[1]; - result.field = fieldAndJoin.field; - result.join = fieldAndJoin.join; + info.path = split[0]; + info.prefix = prefixPos > 0 ? info.path.substring(0, prefixPos) : ''; + info.suffix = !split[1] ? '' : ':' + split[1]; + info.field = fieldAndJoin.field; + info.join = fieldAndJoin.join; + info.alias = splitAs[1] || (info.fn ? info.fn.name + ':' + info.path + info.suffix : split[0]); } - return result; + return info; } return { getEntity: getEntity, diff --git a/ext/search/ang/crmSearchAdmin/crmSearchAdmin.component.js b/ext/search/ang/crmSearchAdmin/crmSearchAdmin.component.js index a162ca4ed61d..36578e224e02 100644 --- a/ext/search/ang/crmSearchAdmin/crmSearchAdmin.component.js +++ b/ext/search/ang/crmSearchAdmin/crmSearchAdmin.component.js @@ -373,7 +373,7 @@ if (ctrl.savedSearch.api_params.groupBy.length) { _.each(ctrl.savedSearch.api_params.select, function(col, pos) { if (!_.contains(col, '(') && ctrl.canAggregate(col)) { - ctrl.savedSearch.api_params.select[pos] = ctrl.DEFAULT_AGGREGATE_FN + '(' + col + ')'; + ctrl.savedSearch.api_params.select[pos] = ctrl.DEFAULT_AGGREGATE_FN + '(DISTINCT ' + col + ')'; } }); } @@ -578,8 +578,7 @@ $scope.formatResult = function(row, col) { var info = searchMeta.parseExpr(col), - key = info.fn ? (info.fn.name + ':' + info.path + info.suffix) : col, - value = row[key]; + value = row[info.alias]; if (info.fn && info.fn.name === 'COUNT') { return value; } diff --git a/ext/search/ang/crmSearchAdmin/crmSearchFunction.component.js b/ext/search/ang/crmSearchAdmin/crmSearchFunction.component.js index ff009085f8e7..0f0e5d2b909b 100644 --- a/ext/search/ang/crmSearchAdmin/crmSearchFunction.component.js +++ b/ext/search/ang/crmSearchAdmin/crmSearchFunction.component.js @@ -37,8 +37,13 @@ ctrl.writeExpr(); }; + // Make a sql-friendly alias for this expression + function makeAlias() { + return (ctrl.fn + '_' + (ctrl.modifier ? ctrl.modifier + '_' : '') + ctrl.path).replace(/[.:]/g, '_'); + } + this.writeExpr = function() { - ctrl.expr = ctrl.fn ? (ctrl.fn + '(' + (ctrl.modifier ? ctrl.modifier + ' ' : '') + ctrl.path + ')') : ctrl.path; + ctrl.expr = ctrl.fn ? (ctrl.fn + '(' + (ctrl.modifier ? ctrl.modifier + ' ' : '') + ctrl.path + ') AS ' + makeAlias()) : ctrl.path; }; } }); From 18436e43532efa0c52ac038c3b0b510007b45307 Mon Sep 17 00:00:00 2001 From: Seamus Lee Date: Mon, 21 Dec 2020 14:52:46 +1100 Subject: [PATCH 34/63] dev/core#2263 Do not try and store items in the session if the session is currently empty Store the locale in the session even if we are in a single lingual instance --- CRM/Core/BAO/ConfigSetting.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CRM/Core/BAO/ConfigSetting.php b/CRM/Core/BAO/ConfigSetting.php index 54ad2e1a74d1..7fd74c1aeb05 100644 --- a/CRM/Core/BAO/ConfigSetting.php +++ b/CRM/Core/BAO/ConfigSetting.php @@ -205,9 +205,6 @@ public static function applyLocale($settings, $activatedLocales) { $chosenLocale = $defaultLocale; } - // Always assign the chosen locale to the session. - $session->set('lcMessages', $chosenLocale); - } else { @@ -216,6 +213,11 @@ public static function applyLocale($settings, $activatedLocales) { } + if (!$session->isEmpty()) { + // Always assign the chosen locale to the session. + $session->set('lcMessages', $chosenLocale); + } + /* * Set suffix for table names in multi-language installs. * Use views if more than one language. From 95519b124e1e3709952c7033ef13d68e257448d9 Mon Sep 17 00:00:00 2001 From: eileen Date: Mon, 21 Dec 2020 18:26:15 +1300 Subject: [PATCH 35/63] Switch to using shared function to call deprecated function With this done all the functionality in the deprecated function can be moved into the shared function --- CRM/Contribute/Import/Parser/Contribution.php | 6 +----- CRM/Import/Parser.php | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/CRM/Contribute/Import/Parser/Contribution.php b/CRM/Contribute/Import/Parser/Contribution.php index d871a075054d..26c4c47ef796 100644 --- a/CRM/Contribute/Import/Parser/Contribution.php +++ b/CRM/Contribute/Import/Parser/Contribution.php @@ -352,10 +352,6 @@ public function import($onDuplicate, &$values) { } if ($this->_contactIdIndex < 0) { - // set the contact type if its not set - if (!isset($paramValues['contact_type'])) { - $paramValues['contact_type'] = $this->_contactType; - } $error = $this->checkContactDuplicate($paramValues); @@ -911,7 +907,7 @@ private function deprecatedFormatParams($params, &$values, $create = FALSE, $onD } else { // we need to get contribution contact using de dupe - $error = _civicrm_api3_deprecated_check_contact_dedupe($params); + $error = $this->checkContactDuplicate($params); if (isset($error['error_message']['params'][0])) { $matchedIDs = explode(',', $error['error_message']['params'][0]); diff --git a/CRM/Import/Parser.php b/CRM/Import/Parser.php index 3aa731f55f02..05a22b58bba7 100644 --- a/CRM/Import/Parser.php +++ b/CRM/Import/Parser.php @@ -524,7 +524,7 @@ public static function saveFileName($type) { */ protected function checkContactDuplicate(&$formatValues) { //retrieve contact id using contact dedupe rule - $formatValues['contact_type'] = $this->_contactType; + $formatValues['contact_type'] = $formatValues['contact_type'] ?? $this->_contactType; $formatValues['version'] = 3; require_once 'CRM/Utils/DeprecatedUtils.php'; $error = _civicrm_api3_deprecated_check_contact_dedupe($formatValues); From fc30268659af43e264bf2171ac3f03748b069940 Mon Sep 17 00:00:00 2001 From: eileen Date: Mon, 21 Dec 2020 18:31:22 +1300 Subject: [PATCH 36/63] [REF] Relocate function from DeprecatedUtils to the class that actually calls it --- CRM/Contact/Import/Parser/Contact.php | 54 ++++++++++++++++++++++++++- CRM/Utils/DeprecatedUtils.php | 52 -------------------------- 2 files changed, 53 insertions(+), 53 deletions(-) diff --git a/CRM/Contact/Import/Parser/Contact.php b/CRM/Contact/Import/Parser/Contact.php index 6908aadc2fe7..20334b7f92c6 100644 --- a/CRM/Contact/Import/Parser/Contact.php +++ b/CRM/Contact/Import/Parser/Contact.php @@ -1565,7 +1565,7 @@ public function createContact(&$formatted, &$contactFields, $onDuplicate, $conta if ($error) { return $error; } - _civicrm_api3_deprecated_validate_formatted_contact($formatted); + $this->deprecated_validate_formatted_contact($formatted); } catch (CRM_Core_Exception $e) { return ['error_message' => $e->getMessage(), 'is_error' => 1, 'code' => $e->getCode()]; @@ -2070,4 +2070,56 @@ protected function handleDuplicateError(array $newContact, $statusFieldName, arr return $this->processMessage($values, $statusFieldName, CRM_Import_Parser::VALID); } + /** + * Validate a formatted contact parameter list. + * + * @param array $params + * Structured parameter list (as in crm_format_params). + * + * @throw CRM_Core_Error + */ + public function deprecated_validate_formatted_contact(&$params): void { + // Look for offending email addresses + + if (array_key_exists('email', $params)) { + foreach ($params['email'] as $count => $values) { + if (!is_array($values)) { + continue; + } + if ($email = CRM_Utils_Array::value('email', $values)) { + // validate each email + if (!CRM_Utils_Rule::email($email)) { + throw new CRM_Core_Exception('No valid email address'); + } + + // check for loc type id. + if (empty($values['location_type_id'])) { + throw new CRM_Core_Exception('Location Type Id missing.'); + } + } + } + } + + // Validate custom data fields + if (array_key_exists('custom', $params) && is_array($params['custom'])) { + foreach ($params['custom'] as $key => $custom) { + if (is_array($custom)) { + foreach ($custom as $fieldId => $value) { + $valid = CRM_Core_BAO_CustomValue::typecheck(CRM_Utils_Array::value('type', $value), + CRM_Utils_Array::value('value', $value) + ); + if (!$valid && $value['is_required']) { + throw new CRM_Core_Exception('Invalid value for custom field \'' . + $custom['name'] . '\'' + ); + } + if (CRM_Utils_Array::value('type', $custom) == 'Date') { + $params['custom'][$key][$fieldId]['value'] = str_replace('-', '', $params['custom'][$key][$fieldId]['value']); + } + } + } + } + } + } + } diff --git a/CRM/Utils/DeprecatedUtils.php b/CRM/Utils/DeprecatedUtils.php index c9345e39adae..8f8b27cf4c5b 100644 --- a/CRM/Utils/DeprecatedUtils.php +++ b/CRM/Utils/DeprecatedUtils.php @@ -551,58 +551,6 @@ function _civicrm_api3_deprecated_duplicate_formatted_contact($params) { return civicrm_api3_create_success(TRUE); } -/** - * Validate a formatted contact parameter list. - * - * @param array $params - * Structured parameter list (as in crm_format_params). - * - * @throw CRM_Core_Error - */ -function _civicrm_api3_deprecated_validate_formatted_contact(&$params): void { - // Look for offending email addresses - - if (array_key_exists('email', $params)) { - foreach ($params['email'] as $count => $values) { - if (!is_array($values)) { - continue; - } - if ($email = CRM_Utils_Array::value('email', $values)) { - // validate each email - if (!CRM_Utils_Rule::email($email)) { - throw new CRM_Core_Exception('No valid email address'); - } - - // check for loc type id. - if (empty($values['location_type_id'])) { - throw new CRM_Core_Exception('Location Type Id missing.'); - } - } - } - } - - // Validate custom data fields - if (array_key_exists('custom', $params) && is_array($params['custom'])) { - foreach ($params['custom'] as $key => $custom) { - if (is_array($custom)) { - foreach ($custom as $fieldId => $value) { - $valid = CRM_Core_BAO_CustomValue::typecheck(CRM_Utils_Array::value('type', $value), - CRM_Utils_Array::value('value', $value) - ); - if (!$valid && $value['is_required']) { - throw new CRM_Core_Exception('Invalid value for custom field \'' . - $custom['name'] . '\'' - ); - } - if (CRM_Utils_Array::value('type', $custom) == 'Date') { - $params['custom'][$key][$fieldId]['value'] = str_replace('-', '', $params['custom'][$key][$fieldId]['value']); - } - } - } - } - } -} - /** * @deprecated - this is part of the import parser not the API & needs to be moved on out * From 4a50b17ff4fc3d4a41974f874a5ae714947ca872 Mon Sep 17 00:00:00 2001 From: eileen Date: Mon, 21 Dec 2020 20:54:20 +1300 Subject: [PATCH 37/63] [REF] Relocate another function from DeprecatedUtils to the calling class This is only called from one class so relocate to there --- CRM/Activity/Import/Parser/Activity.php | 66 ++++++++++++++++++++++++- CRM/Utils/DeprecatedUtils.php | 63 ----------------------- 2 files changed, 64 insertions(+), 65 deletions(-) diff --git a/CRM/Activity/Import/Parser/Activity.php b/CRM/Activity/Import/Parser/Activity.php index aa2239dc3da8..cccd68e0dbfe 100644 --- a/CRM/Activity/Import/Parser/Activity.php +++ b/CRM/Activity/Import/Parser/Activity.php @@ -271,8 +271,7 @@ public function import($onDuplicate, &$values) { } } // Date-Format part ends. - require_once 'CRM/Utils/DeprecatedUtils.php'; - $formatError = _civicrm_api3_deprecated_activity_formatted_param($params, $params, TRUE); + $formatError = $this->deprecated_activity_formatted_param($params, $params, TRUE); if ($formatError) { array_unshift($values, $formatError['error_message']); @@ -374,4 +373,67 @@ public function import($onDuplicate, &$values) { return CRM_Import_Parser::VALID; } + /** + * take the input parameter list as specified in the data model and + * convert it into the same format that we use in QF and BAO object + * + * @param array $params + * Associative array of property name/value. + * pairs to insert in new contact. + * @param array $values + * The reformatted properties that we can use internally. + * + * @param array|bool $create Is the formatted Values array going to + * be used for CRM_Activity_BAO_Activity::create() + * + * @return array|CRM_Error + */ + protected function deprecated_activity_formatted_param(&$params, &$values, $create = FALSE) { + // copy all the activity fields as is + $fields = CRM_Activity_DAO_Activity::fields(); + _civicrm_api3_store_values($fields, $params, $values); + + require_once 'CRM/Core/OptionGroup.php'; + $customFields = CRM_Core_BAO_CustomField::getFields('Activity'); + + foreach ($params as $key => $value) { + // ignore empty values or empty arrays etc + if (CRM_Utils_System::isNull($value)) { + continue; + } + + //Handling Custom Data + if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) { + $values[$key] = $value; + $type = $customFields[$customFieldID]['html_type']; + if (CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID])) { + $values[$key] = CRM_Import_Parser::unserializeCustomValue($customFieldID, $value, $type); + } + elseif ($type == 'Select' || $type == 'Radio') { + $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE); + foreach ($customOption as $customFldID => $customValue) { + $val = $customValue['value'] ?? NULL; + $label = $customValue['label'] ?? NULL; + $label = strtolower($label); + $value = strtolower(trim($value)); + if (($value == $label) || ($value == strtolower($val))) { + $values[$key] = $val; + } + } + } + } + + if ($key == 'target_contact_id') { + if (!CRM_Utils_Rule::integer($value)) { + return civicrm_api3_create_error("contact_id not valid: $value"); + } + $contactID = CRM_Core_DAO::singleValueQuery("SELECT id FROM civicrm_contact WHERE id = $value"); + if (!$contactID) { + return civicrm_api3_create_error("Invalid Contact ID: There is no contact record with contact_id = $value."); + } + } + } + return NULL; + } + } diff --git a/CRM/Utils/DeprecatedUtils.php b/CRM/Utils/DeprecatedUtils.php index 8f8b27cf4c5b..1fe35cc8da1e 100644 --- a/CRM/Utils/DeprecatedUtils.php +++ b/CRM/Utils/DeprecatedUtils.php @@ -101,69 +101,6 @@ function _civicrm_api3_deprecated_check_contact_dedupe($params) { return _civicrm_api3_deprecated_duplicate_formatted_contact($contactFormatted); } -/** - * take the input parameter list as specified in the data model and - * convert it into the same format that we use in QF and BAO object - * - * @param array $params - * Associative array of property name/value. - * pairs to insert in new contact. - * @param array $values - * The reformatted properties that we can use internally. - * - * @param array|bool $create Is the formatted Values array going to - * be used for CRM_Activity_BAO_Activity::create() - * - * @return array|CRM_Error - */ -function _civicrm_api3_deprecated_activity_formatted_param(&$params, &$values, $create = FALSE) { - // copy all the activity fields as is - $fields = CRM_Activity_DAO_Activity::fields(); - _civicrm_api3_store_values($fields, $params, $values); - - require_once 'CRM/Core/OptionGroup.php'; - $customFields = CRM_Core_BAO_CustomField::getFields('Activity'); - - foreach ($params as $key => $value) { - // ignore empty values or empty arrays etc - if (CRM_Utils_System::isNull($value)) { - continue; - } - - //Handling Custom Data - if ($customFieldID = CRM_Core_BAO_CustomField::getKeyID($key)) { - $values[$key] = $value; - $type = $customFields[$customFieldID]['html_type']; - if (CRM_Core_BAO_CustomField::isSerialized($customFields[$customFieldID])) { - $values[$key] = CRM_Import_Parser::unserializeCustomValue($customFieldID, $value, $type); - } - elseif ($type == 'Select' || $type == 'Radio') { - $customOption = CRM_Core_BAO_CustomOption::getCustomOption($customFieldID, TRUE); - foreach ($customOption as $customFldID => $customValue) { - $val = $customValue['value'] ?? NULL; - $label = $customValue['label'] ?? NULL; - $label = strtolower($label); - $value = strtolower(trim($value)); - if (($value == $label) || ($value == strtolower($val))) { - $values[$key] = $val; - } - } - } - } - - if ($key == 'target_contact_id') { - if (!CRM_Utils_Rule::integer($value)) { - return civicrm_api3_create_error("contact_id not valid: $value"); - } - $contactID = CRM_Core_DAO::singleValueQuery("SELECT id FROM civicrm_contact WHERE id = $value"); - if (!$contactID) { - return civicrm_api3_create_error("Invalid Contact ID: There is no contact record with contact_id = $value."); - } - } - } - return NULL; -} - /** * This function adds the contact variable in $values to the * parameter list $params. For most cases, $values should have length 1. If From d463c543e5b25e8782a616042082d21e0fd2ec6a Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Mon, 21 Dec 2020 00:10:24 -0800 Subject: [PATCH 38/63] (dev/core#2258) CryptoRegistry - Allow multiple plaintext entries This allows the plaintext entries to do have different prioritizations among different keys. --- Civi/Crypto/CryptoRegistry.php | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/Civi/Crypto/CryptoRegistry.php b/Civi/Crypto/CryptoRegistry.php index 0cca04fe8912..21e1d103cf2c 100644 --- a/Civi/Crypto/CryptoRegistry.php +++ b/Civi/Crypto/CryptoRegistry.php @@ -68,10 +68,13 @@ public static function createDefaultRegistry() { $registry->addPlainText(['tags' => ['CRED']]); if (defined('CIVICRM_CRED_KEYS')) { foreach (explode(' ', CIVICRM_CRED_KEYS) as $n => $keyExpr) { - $registry->addSymmetricKey($registry->parseKey($keyExpr) + [ - 'tags' => ['CRED'], - 'weight' => $n, - ]); + $key = ['tags' => ['CRED'], 'weight' => $n]; + if ($keyExpr === 'plain') { + $registry->addPlainText($key); + } + else { + $registry->addSymmetricKey($registry->parseKey($keyExpr) + $key); + } } } @@ -169,14 +172,15 @@ public function isValidKeyId($id) { * @return array */ public function addPlainText($options) { - if (!isset($this->keys['plain'])) { - } - if (isset($options['tags'])) { - $this->keys['plain']['tags'] = array_merge( - $options['tags'] - ); - } - return $this->keys['plain']; + static $n = 0; + $defaults = [ + 'suite' => 'plain', + 'weight' => self::LAST_WEIGHT, + ]; + $options = array_merge($defaults, $options); + $options['id'] = 'plain' . ($n++); + $this->keys[$options['id']] = $options; + return $options; } /** From 8d54915f6e38f4e786f0a3839ba7d48fb8306000 Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Mon, 21 Dec 2020 00:15:27 -0800 Subject: [PATCH 39/63] (dev/core#2258) CryptoRegistry - Add findKeysByTag() --- Civi/Crypto/CryptoRegistry.php | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Civi/Crypto/CryptoRegistry.php b/Civi/Crypto/CryptoRegistry.php index 21e1d103cf2c..c979a257d63e 100644 --- a/Civi/Crypto/CryptoRegistry.php +++ b/Civi/Crypto/CryptoRegistry.php @@ -233,6 +233,24 @@ public function findKey($keyIds) { throw new CryptoException("Failed to find key by ID or tag (" . implode(' ', $keyIds) . ")"); } + /** + * Find all the keys that apply to a tag. + * + * @param string $keyTag + * + * @return array + * List of keys, indexed by id, ordered by weight. + */ + public function findKeysByTag($keyTag) { + $keys = array_filter($this->keys, function ($key) use ($keyTag) { + return in_array($keyTag, $key['tags'] ?? []); + }); + uasort($keys, function($a, $b) { + return ($a['weight'] ?? 0) - ($b['weight'] ?? 0); + }); + return $keys; + } + /** * @param string $name * @return \Civi\Crypto\CipherSuiteInterface From a5eae9cb731d6298112d64f55e5d6d7f3bc8968b Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Sun, 20 Dec 2020 13:28:17 -0800 Subject: [PATCH 40/63] (REF) CryptoToken - Extract method 'parse()' --- Civi/Crypto/CryptoToken.php | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/Civi/Crypto/CryptoToken.php b/Civi/Crypto/CryptoToken.php index 038f405ce2d2..2a478fd7ce4e 100644 --- a/Civi/Crypto/CryptoToken.php +++ b/Civi/Crypto/CryptoToken.php @@ -130,27 +130,40 @@ public function decrypt($token, $keyIdOrTag = '*') { /** @var CryptoRegistry $registry */ $registry = \Civi::service('crypto.registry'); + $tokenData = $this->parse($token); + + $key = $registry->findKey($tokenData['k']); + if (!in_array('*', $keyIdOrTag) && !in_array($tokenData['k'], $keyIdOrTag) && empty(array_intersect($keyIdOrTag, $key['tags']))) { + throw new CryptoException("Cannot decrypt token. Unexpected key: {$tokenData['k']}"); + } + + /** @var \Civi\Crypto\CipherSuiteInterface $cipherSuite */ + $cipherSuite = $registry->findSuite($key['suite']); + $plainText = $cipherSuite->decrypt($tokenData['t'], $key); + return $plainText; + } + + /** + * Parse the content of a token (without decrypting it). + * + * @param string $token + * + * @return array + * @throws \Civi\Crypto\Exception\CryptoException + */ + public function parse($token): array { $fmt = substr($token, 1, 4); switch ($fmt) { case self::FMT_QUERY: + $tokenData = []; parse_str(substr($token, 5), $tokenData); - $keyId = $tokenData['k']; - $cipherText = \CRM_Utils_String::base64UrlDecode($tokenData['t']); + $tokenData['t'] = \CRM_Utils_String::base64UrlDecode($tokenData['t']); break; default: throw new CryptoException("Cannot decrypt token. Invalid format."); } - - $key = $registry->findKey($keyId); - if (!in_array('*', $keyIdOrTag) && !in_array($keyId, $keyIdOrTag) && empty(array_intersect($keyIdOrTag, $key['tags']))) { - throw new CryptoException("Cannot decrypt token. Unexpected key: {$keyId}"); - } - - /** @var \Civi\Crypto\CipherSuiteInterface $cipherSuite */ - $cipherSuite = $registry->findSuite($key['suite']); - $plainText = $cipherSuite->decrypt($cipherText, $key); - return $plainText; + return $tokenData; } } From ec089f4bf3621b12975a7f8926c5af789a46f94d Mon Sep 17 00:00:00 2001 From: Jitendra Purohit Date: Mon, 21 Dec 2020 16:30:25 +0530 Subject: [PATCH 41/63] Fix fatal error on contribution summary report -> add to group action --- CRM/Report/Form.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CRM/Report/Form.php b/CRM/Report/Form.php index b70d4869e6ac..ceaca5ef3904 100644 --- a/CRM/Report/Form.php +++ b/CRM/Report/Form.php @@ -4962,7 +4962,11 @@ public function add2group($groupID) { $select = preg_replace('/SELECT(\s+SQL_CALC_FOUND_ROWS)?\s+/i', $select, $this->_select); $sql = "{$select} {$this->_from} {$this->_where} {$this->_groupBy} {$this->_having} {$this->_orderBy}"; $sql = str_replace('WITH ROLLUP', '', $sql); + if (!$this->optimisedForOnlyFullGroupBy) { + CRM_Core_DAO::disableFullGroupByMode(); + } $dao = CRM_Core_DAO::executeQuery($sql); + CRM_Core_DAO::reenableFullGroupByMode(); $contact_ids = []; // Add resulting contacts to group From 56af1071ddf46f08cf4dc7b5d3c5d0c2db846197 Mon Sep 17 00:00:00 2001 From: Matthew Wire Date: Mon, 21 Dec 2020 17:16:25 +0000 Subject: [PATCH 42/63] Improve logging when a contribution is created/updated --- CRM/Contribute/BAO/Contribution.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CRM/Contribute/BAO/Contribution.php b/CRM/Contribute/BAO/Contribution.php index 95574696cb63..0d7b5cb24ffa 100644 --- a/CRM/Contribute/BAO/Contribution.php +++ b/CRM/Contribute/BAO/Contribution.php @@ -4254,8 +4254,8 @@ public static function completeOrder($input, $ids, $contribution, $isPostPayment $contribution->contribution_status_id = $contributionParams['contribution_status_id']; - CRM_Core_Error::debug_log_message('Contribution record updated successfully'); $transaction->commit(); + \Civi::log()->info("Contribution {$contributionParams['id']} updated successfully"); // @todo - check if Contribution::create does this, test, remove. CRM_Contribute_BAO_ContributionRecur::updateRecurLinkedPledge($contributionID, $recurringContributionID, @@ -4278,10 +4278,9 @@ public static function completeOrder($input, $ids, $contribution, $isPostPayment 'id' => $contributionID, 'payment_processor_id' => $paymentProcessorId, ]); - CRM_Core_Error::debug_log_message("Receipt sent"); + \Civi::log()->info("Contribution {$contributionParams['id']} Receipt sent"); } - CRM_Core_Error::debug_log_message("Success: Database updated"); return $contributionResult; } From 4abeee38124c12fe1c933f1fe1cd1c1be101e03c Mon Sep 17 00:00:00 2001 From: eileen Date: Mon, 21 Dec 2020 20:39:36 +1300 Subject: [PATCH 43/63] [REF] Relocate another deprecated utils function to the only class that calls it. I removed the tests because they didn't test much & what they did test seemed like stuff we should alter --- CRM/Contact/Import/Parser/Contact.php | 114 +++++++++++++++++- CRM/Utils/DeprecatedUtils.php | 112 ----------------- .../phpunit/CRM/Utils/DeprecatedUtilsTest.php | 82 ------------- 3 files changed, 113 insertions(+), 195 deletions(-) delete mode 100644 tests/phpunit/CRM/Utils/DeprecatedUtilsTest.php diff --git a/CRM/Contact/Import/Parser/Contact.php b/CRM/Contact/Import/Parser/Contact.php index 20334b7f92c6..b35004675b33 100644 --- a/CRM/Contact/Import/Parser/Contact.php +++ b/CRM/Contact/Import/Parser/Contact.php @@ -1561,7 +1561,7 @@ public function createContact(&$formatted, &$contactFields, $onDuplicate, $conta // setting required check to false, CRM-2839 // plus we do our own required check in import try { - $error = _civicrm_api3_deprecated_contact_check_params($formatted, $dupeCheck, $dedupeRuleGroupID); + $error = $this->deprecated_contact_check_params($formatted, $dupeCheck, $dedupeRuleGroupID); if ($error) { return $error; } @@ -2122,4 +2122,116 @@ public function deprecated_validate_formatted_contact(&$params): void { } } + /** + * @param array $params + * @param bool $dupeCheck + * @param null|int $dedupeRuleGroupID + * + * @throws \CRM_Core_Exception + */ + public function deprecated_contact_check_params( + &$params, + $dupeCheck = TRUE, + $dedupeRuleGroupID = NULL) { + + $requiredCheck = TRUE; + + if (isset($params['id']) && is_numeric($params['id'])) { + $requiredCheck = FALSE; + } + if ($requiredCheck) { + if (isset($params['id'])) { + $required = ['Individual', 'Household', 'Organization']; + } + $required = [ + 'Individual' => [ + ['first_name', 'last_name'], + 'email', + ], + 'Household' => [ + 'household_name', + ], + 'Organization' => [ + 'organization_name', + ], + ]; + + // contact_type has a limited number of valid values + if (empty($params['contact_type'])) { + throw new CRM_Core_Exception("No Contact Type"); + } + $fields = $required[$params['contact_type']] ?? NULL; + if ($fields == NULL) { + throw new CRM_Core_Exception("Invalid Contact Type: {$params['contact_type']}"); + } + + if ($csType = CRM_Utils_Array::value('contact_sub_type', $params)) { + if (!(CRM_Contact_BAO_ContactType::isExtendsContactType($csType, $params['contact_type']))) { + throw new CRM_Core_Exception("Invalid or Mismatched Contact Subtype: " . implode(', ', (array) $csType)); + } + } + + if (empty($params['contact_id']) && !empty($params['id'])) { + $valid = FALSE; + $error = ''; + foreach ($fields as $field) { + if (is_array($field)) { + $valid = TRUE; + foreach ($field as $element) { + if (empty($params[$element])) { + $valid = FALSE; + $error .= $element; + break; + } + } + } + else { + if (!empty($params[$field])) { + $valid = TRUE; + } + } + if ($valid) { + break; + } + } + + if (!$valid) { + throw new CRM_Core_Exception("Required fields not found for {$params['contact_type']} : $error"); + } + } + } + + if ($dupeCheck) { + // @todo switch to using api version + // $dupes = civicrm_api3('Contact', 'duplicatecheck', (array('match' => $params, 'dedupe_rule_id' => $dedupeRuleGroupID))); + // $ids = $dupes['count'] ? implode(',', array_keys($dupes['values'])) : NULL; + $ids = CRM_Contact_BAO_Contact::getDuplicateContacts($params, $params['contact_type'], 'Unsupervised', [], CRM_Utils_Array::value('check_permissions', $params), $dedupeRuleGroupID); + if ($ids != NULL) { + $error = CRM_Core_Error::createError("Found matching contacts: " . implode(',', $ids), + CRM_Core_Error::DUPLICATE_CONTACT, + 'Fatal', $ids + ); + return civicrm_api3_create_error($error->pop()); + } + } + + // check for organisations with same name + if (!empty($params['current_employer'])) { + $organizationParams = ['organization_name' => $params['current_employer']]; + $dupeIds = CRM_Contact_BAO_Contact::getDuplicateContacts($organizationParams, 'Organization', 'Supervised', [], FALSE); + + // check for mismatch employer name and id + if (!empty($params['employer_id']) && !in_array($params['employer_id'], $dupeIds) + ) { + throw new CRM_Core_Exception('Employer name and Employer id Mismatch'); + } + + // show error if multiple organisation with same name exist + if (empty($params['employer_id']) && (count($dupeIds) > 1) + ) { + return civicrm_api3_create_error('Found more than one Organisation with same Name.'); + } + } + } + } diff --git a/CRM/Utils/DeprecatedUtils.php b/CRM/Utils/DeprecatedUtils.php index 1fe35cc8da1e..e614413bc13e 100644 --- a/CRM/Utils/DeprecatedUtils.php +++ b/CRM/Utils/DeprecatedUtils.php @@ -568,118 +568,6 @@ function _civicrm_api3_deprecated_participant_check_params($params, $checkDuplic return TRUE; } -/** - * @param array $params - * @param bool $dupeCheck - * @param null|int $dedupeRuleGroupID - * - * @throws \CRM_Core_Exception - */ -function _civicrm_api3_deprecated_contact_check_params( - &$params, - $dupeCheck = TRUE, - $dedupeRuleGroupID = NULL) { - - $requiredCheck = TRUE; - - if (isset($params['id']) && is_numeric($params['id'])) { - $requiredCheck = FALSE; - } - if ($requiredCheck) { - if (isset($params['id'])) { - $required = ['Individual', 'Household', 'Organization']; - } - $required = [ - 'Individual' => [ - ['first_name', 'last_name'], - 'email', - ], - 'Household' => [ - 'household_name', - ], - 'Organization' => [ - 'organization_name', - ], - ]; - - // contact_type has a limited number of valid values - if (empty($params['contact_type'])) { - throw new CRM_Core_Exception("No Contact Type"); - } - $fields = $required[$params['contact_type']] ?? NULL; - if ($fields == NULL) { - throw new CRM_Core_Exception("Invalid Contact Type: {$params['contact_type']}"); - } - - if ($csType = CRM_Utils_Array::value('contact_sub_type', $params)) { - if (!(CRM_Contact_BAO_ContactType::isExtendsContactType($csType, $params['contact_type']))) { - throw new CRM_Core_Exception("Invalid or Mismatched Contact Subtype: " . implode(', ', (array) $csType)); - } - } - - if (empty($params['contact_id']) && !empty($params['id'])) { - $valid = FALSE; - $error = ''; - foreach ($fields as $field) { - if (is_array($field)) { - $valid = TRUE; - foreach ($field as $element) { - if (empty($params[$element])) { - $valid = FALSE; - $error .= $element; - break; - } - } - } - else { - if (!empty($params[$field])) { - $valid = TRUE; - } - } - if ($valid) { - break; - } - } - - if (!$valid) { - throw new CRM_Core_Exception("Required fields not found for {$params['contact_type']} : $error"); - } - } - } - - if ($dupeCheck) { - // @todo switch to using api version - // $dupes = civicrm_api3('Contact', 'duplicatecheck', (array('match' => $params, 'dedupe_rule_id' => $dedupeRuleGroupID))); - // $ids = $dupes['count'] ? implode(',', array_keys($dupes['values'])) : NULL; - $ids = CRM_Contact_BAO_Contact::getDuplicateContacts($params, $params['contact_type'], 'Unsupervised', [], CRM_Utils_Array::value('check_permissions', $params), $dedupeRuleGroupID); - if ($ids != NULL) { - $error = CRM_Core_Error::createError("Found matching contacts: " . implode(',', $ids), - CRM_Core_Error::DUPLICATE_CONTACT, - 'Fatal', $ids - ); - return civicrm_api3_create_error($error->pop()); - } - } - - // check for organisations with same name - if (!empty($params['current_employer'])) { - $organizationParams = ['organization_name' => $params['current_employer']]; - $dupeIds = CRM_Contact_BAO_Contact::getDuplicateContacts($organizationParams, 'Organization', 'Supervised', [], FALSE); - - // check for mismatch employer name and id - if (!empty($params['employer_id']) && !in_array($params['employer_id'], $dupeIds) - ) { - throw new CRM_Core_Exception('Employer name and Employer id Mismatch'); - } - - // show error if multiple organisation with same name exist - if (empty($params['employer_id']) && (count($dupeIds) > 1) - ) { - return civicrm_api3_create_error('Found more than one Organisation with same Name.'); - } - } -} - /** * @param $result * @param int $activityTypeID diff --git a/tests/phpunit/CRM/Utils/DeprecatedUtilsTest.php b/tests/phpunit/CRM/Utils/DeprecatedUtilsTest.php deleted file mode 100644 index 3091598b403b..000000000000 --- a/tests/phpunit/CRM/Utils/DeprecatedUtilsTest.php +++ /dev/null @@ -1,82 +0,0 @@ -quickCleanup($tablesToTruncate); - } - - /** - * Test civicrm_contact_check_params with no contact type. - */ - public function testCheckParamsWithNoContactType(): void { - $params = ['foo' => 'bar']; - try { - _civicrm_api3_deprecated_contact_check_params($params, FALSE); - } - catch (CRM_Core_Exception $e) { - return; - } - $this->fail('An exception should have been thrown'); - } - - /** - * Test civicrm_contact_check_params with a duplicate. - * - * @throws \CiviCRM_API3_Exception - */ - public function testCheckParamsWithDuplicateContact2(): void { - $this->individualCreate(['first_name' => 'Test', 'last_name' => 'Contact', 'email' => 'TestContact@example.com']); - - $params = [ - 'first_name' => 'Test', - 'last_name' => 'Contact', - 'email' => 'TestContact@example.com', - 'contact_type' => 'Individual', - ]; - try { - $error = _civicrm_api3_deprecated_contact_check_params($params, TRUE); - $this->assertEquals(1, $error['is_error']); - } - catch (CRM_Core_Exception $e) { - $this->assertRegexp("/matching contacts.*1/s", - $e->getMessage() - ); - return; - } - // $this->fail('Exception was not optional'); - } - - /** - * Test civicrm_contact_check_params with check for required params. - */ - public function testCheckParamsWithNoParams(): void { - $params = []; - try { - _civicrm_api3_deprecated_contact_check_params($params, FALSE); - } - catch (CRM_Core_Exception $e) { - return; - } - $this->fail('Exception required'); - } - -} From 880585a2d0148f14d7f567525ab14ecd5f6c5f63 Mon Sep 17 00:00:00 2001 From: eileen Date: Tue, 22 Dec 2020 09:16:03 +1300 Subject: [PATCH 44/63] [REF] Move another deprecated function to the class that uses it --- CRM/Event/Import/Parser/Participant.php | 24 ++++++++++++++++++++++-- CRM/Utils/DeprecatedUtils.php | 22 ---------------------- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/CRM/Event/Import/Parser/Participant.php b/CRM/Event/Import/Parser/Participant.php index 5ed2fecfbc0b..3bb21269c1c4 100644 --- a/CRM/Event/Import/Parser/Participant.php +++ b/CRM/Event/Import/Parser/Participant.php @@ -386,7 +386,7 @@ public function import($onDuplicate, &$values) { foreach ($matchedIDs as $contactId) { $formatted['contact_id'] = $contactId; $formatted['version'] = 3; - $newParticipant = _civicrm_api3_deprecated_create_participant_formatted($formatted, $onDuplicate); + $newParticipant = $this->deprecated_create_participant_formatted($formatted, $onDuplicate); } } } @@ -435,7 +435,7 @@ public function import($onDuplicate, &$values) { } } - $newParticipant = _civicrm_api3_deprecated_create_participant_formatted($formatted, $onDuplicate); + $newParticipant = $this->deprecated_create_participant_formatted($formatted, $onDuplicate); } if (is_array($newParticipant) && civicrm_error($newParticipant)) { @@ -625,4 +625,24 @@ protected function formatValues(&$values, $params) { return NULL; } + /** + * @deprecated - this is part of the import parser not the API & needs to be moved on out + * + * @param array $params + * @param $onDuplicate + * + * @return array|bool + * + */ + protected function deprecated_create_participant_formatted($params, $onDuplicate) { + if ($onDuplicate != CRM_Import_Parser::DUPLICATE_NOCHECK) { + CRM_Core_Error::reset(); + $error = _civicrm_api3_deprecated_participant_check_params($params, TRUE); + if (civicrm_error($error)) { + return $error; + } + } + return civicrm_api3_participant_create($params); + } + } diff --git a/CRM/Utils/DeprecatedUtils.php b/CRM/Utils/DeprecatedUtils.php index 1fe35cc8da1e..ae4efc5b2bc7 100644 --- a/CRM/Utils/DeprecatedUtils.php +++ b/CRM/Utils/DeprecatedUtils.php @@ -488,28 +488,6 @@ function _civicrm_api3_deprecated_duplicate_formatted_contact($params) { return civicrm_api3_create_success(TRUE); } -/** - * @deprecated - this is part of the import parser not the API & needs to be moved on out - * - * @param array $params - * @param $onDuplicate - * - * @return array|bool - * - */ -function _civicrm_api3_deprecated_create_participant_formatted($params, $onDuplicate) { - require_once 'CRM/Event/Import/Parser.php'; - if ($onDuplicate != CRM_Import_Parser::DUPLICATE_NOCHECK) { - CRM_Core_Error::reset(); - $error = _civicrm_api3_deprecated_participant_check_params($params, TRUE); - if (civicrm_error($error)) { - return $error; - } - } - require_once "api/v3/Participant.php"; - return civicrm_api3_participant_create($params); -} - /** * * @param array $params From 725a91cb00964958c267b73da455f8f8b1d3b9f4 Mon Sep 17 00:00:00 2001 From: Seamus Lee Date: Tue, 22 Dec 2020 12:19:38 +1100 Subject: [PATCH 45/63] [APIv4] Permit using other SQL functions such as CONCAT within a GROUP_CONCAT --- Civi/Api4/Query/SqlFunctionGROUP_CONCAT.php | 2 +- tests/phpunit/api/v4/Action/SqlFunctionTest.php | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Civi/Api4/Query/SqlFunctionGROUP_CONCAT.php b/Civi/Api4/Query/SqlFunctionGROUP_CONCAT.php index b683f15a7236..fb618e0ac7c0 100644 --- a/Civi/Api4/Query/SqlFunctionGROUP_CONCAT.php +++ b/Civi/Api4/Query/SqlFunctionGROUP_CONCAT.php @@ -24,7 +24,7 @@ class SqlFunctionGROUP_CONCAT extends SqlFunction { [ 'prefix' => ['', 'DISTINCT', 'ALL'], 'expr' => 1, - 'must_be' => ['SqlField'], + 'must_be' => ['SqlField', 'sqlFunction'], 'optional' => FALSE, ], [ diff --git a/tests/phpunit/api/v4/Action/SqlFunctionTest.php b/tests/phpunit/api/v4/Action/SqlFunctionTest.php index 0ff7f2cec13a..27815be6c3b9 100644 --- a/tests/phpunit/api/v4/Action/SqlFunctionTest.php +++ b/tests/phpunit/api/v4/Action/SqlFunctionTest.php @@ -76,6 +76,18 @@ public function testGroupAggregates() { $this->assertTrue(4 === $agg['count']); $this->assertContains('Donation', $agg['GROUP_CONCAT:financial_type_id:name']); + + // Test GROUP_CONCAT with a CONCAT as well + $agg = Contribution::get(FALSE) + ->addGroupBy('contact_id') + ->addWhere('contact_id', '=', $cid) + ->addSelect("GROUP_CONCAT(CONCAT(financial_type_id, ', ', contact_id, ', ', total_amount))") + ->addSelect('COUNT(*) AS count') + ->execute() + ->first(); + + $this->assertTrue(4 === $agg['count']); + $this->assertContains('1, ' . $cid . ', 100.00', $agg['GROUP_CONCAT:financial_type_id_contact_id_total_amount']); } public function testGroupHaving() { From 10916a956a81c61919dc8bd0a00e6a637c3911df Mon Sep 17 00:00:00 2001 From: Vangelis Pantazis Date: Tue, 22 Dec 2020 15:29:20 +0000 Subject: [PATCH 46/63] Rework on the detection of new vs existing smartgroup --- CRM/Contact/Form/Search/Builder.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CRM/Contact/Form/Search/Builder.php b/CRM/Contact/Form/Search/Builder.php index 0eccbcb5049f..f10f2c199f84 100644 --- a/CRM/Contact/Form/Search/Builder.php +++ b/CRM/Contact/Form/Search/Builder.php @@ -51,7 +51,9 @@ public function preProcess() { // Initialize new form if (!$this->_blockCount) { $this->_blockCount = 4; - $this->set('newBlock', 1); + if (!$this->_ssID) { + $this->set('newBlock', 1); + } } //get the column count From 2a320359f5076fb78cee9109777ddb41a5fb8b65 Mon Sep 17 00:00:00 2001 From: eileen Date: Wed, 23 Dec 2020 14:06:38 +1300 Subject: [PATCH 47/63] [REF] Convert previously shared function to non-static, remove unrelated code There is no facility for creating on_behalf on back office membership forms so this code, copied from the shared function, does not relate. The activity IS created by Contribution.create - the extra code was just to hack it around to doing on_behalf Likewise, the back office membership form does not collect data on contribution custom fields or notes so removing these also makes sense here. --- CRM/Member/Form/Membership.php | 49 ++++------------------------------ 1 file changed, 5 insertions(+), 44 deletions(-) diff --git a/CRM/Member/Form/Membership.php b/CRM/Member/Form/Membership.php index 6d6654cbbff0..6ae16df52b70 100644 --- a/CRM/Member/Form/Membership.php +++ b/CRM/Member/Form/Membership.php @@ -975,7 +975,7 @@ public static function emailReceipt(&$form, &$formValues, &$membership, $customV // & we should aim to move this function to the BAO layer in future. // however, we can assume that the contact_id passed in by the batch // function will be the recipient - list($form->_contributorDisplayName, $form->_contributorEmail) + [$form->_contributorDisplayName, $form->_contributorEmail] = CRM_Contact_BAO_Contact_Location::getEmailDetails($formValues['contact_id']); if (empty($form->_receiptContactId) || $isBatchProcess) { $form->_receiptContactId = $formValues['contact_id']; @@ -1170,7 +1170,7 @@ public function submit() { } // Retrieve the name and email of the current user - this will be the FROM for the receipt email - list($userName) = CRM_Contact_BAO_Contact_Location::getEmailDetails(CRM_Core_Session::getLoggedInContactID()); + [$userName] = CRM_Contact_BAO_Contact_Location::getEmailDetails(CRM_Core_Session::getLoggedInContactID()); //CRM-13981, allow different person as a soft-contributor of chosen type if ($this->_contributorContactID != $this->_contactID) { @@ -1295,7 +1295,7 @@ public function submit() { $financialType->find(TRUE); $this->_params = $formValues; - $contribution = self::processFormContribution($this, + $contribution = $this->processContribution( $paymentParams, NULL, [ @@ -1839,7 +1839,6 @@ protected function getSelectedMembershipLabels(): string { * It's like the contribution create being done here is actively bad and * being fixed later. * - * @param CRM_Core_Form $form * @param array $params * @param array $result * @param array $contributionParams @@ -1862,13 +1861,13 @@ protected function getSelectedMembershipLabels(): string { * @throws \CRM_Core_Exception * @throws \CiviCRM_API3_Exception */ - public static function processFormContribution( - &$form, + protected function processContribution( $params, $result, $contributionParams, $financialType ) { + $form = $this; $transaction = new CRM_Core_Transaction(); $contactID = $contributionParams['contact_id']; @@ -1920,44 +1919,6 @@ public static function processFormContribution( //CRM-13981, processing honor contact into soft-credit contribution CRM_Contribute_BAO_ContributionSoft::processSoftContribution($params, $contribution); - if ($contribution) { - //handle custom data. - $params['contribution_id'] = $contribution->id; - if (!empty($params['custom']) && - is_array($params['custom']) - ) { - CRM_Core_BAO_CustomValueTable::store($params['custom'], 'civicrm_contribution', $contribution->id); - } - } - // Save note - if ($contribution && !empty($params['contribution_note'])) { - $noteParams = [ - 'entity_table' => 'civicrm_contribution', - 'note' => $params['contribution_note'], - 'entity_id' => $contribution->id, - 'contact_id' => $contribution->contact_id, - ]; - - CRM_Core_BAO_Note::add($noteParams, []); - } - - //create contribution activity w/ individual and target - //activity w/ organisation contact id when onbelf, CRM-4027 - $actParams = []; - $targetContactID = NULL; - if (!empty($params['onbehalf_contact_id'])) { - $actParams = [ - 'source_contact_id' => $params['onbehalf_contact_id'], - 'on_behalf' => TRUE, - ]; - $targetContactID = $contribution->contact_id; - } - - // create an activity record - if ($contribution) { - CRM_Activity_BAO_Activity::addActivity($contribution, 'Contribution', $targetContactID, $actParams); - } - $transaction->commit(); return $contribution; } From 26f76772e9815d414e4b74ceb8b4ba682b528f43 Mon Sep 17 00:00:00 2001 From: eileen Date: Wed, 23 Dec 2020 14:35:28 +1300 Subject: [PATCH 48/63] Convert contributionSoft to an array This is a partial of #19096 which requires contributionSoft to be an array & just simplifies that commit (which is still failing tests --- CRM/Contribute/BAO/ContributionSoft.php | 2 +- CRM/Contribute/Form/Contribution/Confirm.php | 27 +++++++++++--------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/CRM/Contribute/BAO/ContributionSoft.php b/CRM/Contribute/BAO/ContributionSoft.php index 670e30adfae7..030b9e8dfce0 100644 --- a/CRM/Contribute/BAO/ContributionSoft.php +++ b/CRM/Contribute/BAO/ContributionSoft.php @@ -609,7 +609,7 @@ protected static function processPCP($pcp, $contribution) { $contributionSoft = self::add($softParams); //Send notification to owner for PCP if ($contributionSoft->pcp_id && empty($pcpId)) { - CRM_Contribute_Form_Contribution_Confirm::pcpNotifyOwner($contribution, $contributionSoft); + CRM_Contribute_Form_Contribution_Confirm::pcpNotifyOwner($contribution, (array) $contributionSoft); } } //Delete PCP against this contribution and create new on submitted PCP information diff --git a/CRM/Contribute/Form/Contribution/Confirm.php b/CRM/Contribute/Form/Contribution/Confirm.php index c920a6421c0e..02f3315ec9a5 100644 --- a/CRM/Contribute/Form/Contribution/Confirm.php +++ b/CRM/Contribute/Form/Contribution/Confirm.php @@ -1227,21 +1227,24 @@ public static function processOnBehalfOrganization(&$behalfOrganization, &$conta * This is used by contribution and also event PCPs. * * @param object $contribution - * @param object $contributionSoft + * @param array $contributionSoft * Contribution object. + * + * @throws \API_Exception + * @throws \CRM_Core_Exception */ - public static function pcpNotifyOwner($contribution, $contributionSoft) { - $params = ['id' => $contributionSoft->pcp_id]; + public static function pcpNotifyOwner($contribution, array $contributionSoft) { + $params = ['id' => $contributionSoft['pcp_id']]; CRM_Core_DAO::commonRetrieve('CRM_PCP_DAO_PCP', $params, $pcpInfo); $ownerNotifyID = CRM_Core_DAO::getFieldValue('CRM_PCP_DAO_PCPBlock', $pcpInfo['pcp_block_id'], 'owner_notify_id'); $ownerNotifyOption = CRM_Core_PseudoConstant::getName('CRM_PCP_DAO_PCPBlock', 'owner_notify_id', $ownerNotifyID); if ($ownerNotifyOption != 'no_notifications' && (($ownerNotifyOption == 'owner_chooses' && - CRM_Core_DAO::getFieldValue('CRM_PCP_DAO_PCP', $contributionSoft->pcp_id, 'is_notify')) || + CRM_Core_DAO::getFieldValue('CRM_PCP_DAO_PCP', $contributionSoft['pcp_id'], 'is_notify')) || $ownerNotifyOption == 'all_owners')) { $pcpInfoURL = CRM_Utils_System::url('civicrm/pcp/info', - "reset=1&id={$contributionSoft->pcp_id}", + "reset=1&id={$contributionSoft['pcp_id']}", TRUE, NULL, FALSE, TRUE ); // set email in the template here @@ -1254,22 +1257,22 @@ public static function pcpNotifyOwner($contribution, $contributionSoft) { if (!$email) { [$donorName, $email] = CRM_Contact_BAO_Contact_Location::getEmailDetails($contribution->contact_id); } - [$ownerName, $ownerEmail] = CRM_Contact_BAO_Contact_Location::getEmailDetails($contributionSoft->contact_id); + [$ownerName, $ownerEmail] = CRM_Contact_BAO_Contact_Location::getEmailDetails($contributionSoft['contact_id']); $tplParams = [ 'page_title' => $pcpInfo['title'], 'receive_date' => $contribution->receive_date, - 'total_amount' => $contributionSoft->amount, + 'total_amount' => $contributionSoft['amount'], 'donors_display_name' => $donorName, 'donors_email' => $email, 'pcpInfoURL' => $pcpInfoURL, - 'is_honor_roll_enabled' => $contributionSoft->pcp_display_in_roll, - 'currency' => $contributionSoft->currency, + 'is_honor_roll_enabled' => $contributionSoft['pcp_display_in_roll'], + 'currency' => $contributionSoft['currency'], ]; $domainValues = CRM_Core_BAO_Domain::getNameAndEmail(); $sendTemplateParams = [ 'groupName' => 'msg_tpl_workflow_contribution', 'valueName' => 'pcp_owner_notify', - 'contactId' => $contributionSoft->contact_id, + 'contactId' => $contributionSoft['contact_id'], 'toEmail' => $ownerEmail, 'toName' => $ownerName, 'from' => "$domainValues[0] <$domainValues[1]>", @@ -1291,7 +1294,7 @@ public static function pcpNotifyOwner($contribution, $contributionSoft) { * * @return array */ - public static function processPcp(&$page, $params) { + public static function processPcp(&$page, $params): array { $params['pcp_made_through_id'] = $page->_pcpId; $page->assign('pcpBlock', TRUE); if (!empty($params['pcp_display_in_roll']) && empty($params['pcp_roll_nickname'])) { @@ -1327,7 +1330,7 @@ public static function processPcp(&$page, $params) { * Line items specifically relating to memberships. */ protected function processMembership($membershipParams, $contactID, $customFieldsFormatted, $fieldTypes, $premiumParams, - $membershipLineItems) { + $membershipLineItems): void { $membershipTypeIDs = (array) $membershipParams['selectMembership']; $membershipTypes = CRM_Member_BAO_Membership::buildMembershipTypeValues($this, $membershipTypeIDs); From c7e966548d6fc7b97c65c419fd2a426f825e6bd5 Mon Sep 17 00:00:00 2001 From: Coleman Watts Date: Mon, 21 Dec 2020 21:45:45 -0500 Subject: [PATCH 49/63] Search kit: Add token selector in display admin UI, allow tokens in tooltips Also includes some cleanup of the crmSearchAdminLinkSelect component, making it more self-contained --- .../crmSearchAdminDisplay.component.js | 15 ------ ... => crmSearchAdminLinkSelect.component.js} | 22 +++++++- .../crmSearchAdminLinkSelect.html | 5 +- .../crmSearchAdminTokenSelect.component.js | 51 +++++++++++++++++++ .../crmSearchAdminTokenSelect.html | 10 ++++ .../searchAdminDisplayList.component.js | 1 - .../displays/searchAdminDisplayList.html | 3 +- .../searchAdminDisplayTable.component.js | 1 - .../displays/searchAdminDisplayTable.html | 3 +- ext/search/ang/crmSearchDisplay.module.js | 21 +++++--- .../crmSearchDisplayList.component.js | 1 + .../crmSearchDisplayListItems.html | 2 +- .../crmSearchDisplayTable.component.js | 1 + .../crmSearchDisplayTable.html | 2 +- 14 files changed, 106 insertions(+), 32 deletions(-) rename ext/search/ang/crmSearchAdmin/{crmSearchAdminLinkSelect.directive.js => crmSearchAdminLinkSelect.component.js} (60%) create mode 100644 ext/search/ang/crmSearchAdmin/crmSearchAdminTokenSelect.component.js create mode 100644 ext/search/ang/crmSearchAdmin/crmSearchAdminTokenSelect.html diff --git a/ext/search/ang/crmSearchAdmin/crmSearchAdminDisplay.component.js b/ext/search/ang/crmSearchAdmin/crmSearchAdminDisplay.component.js index 0def0e277027..8ab7a1cd3035 100644 --- a/ext/search/ang/crmSearchAdmin/crmSearchAdminDisplay.component.js +++ b/ext/search/ang/crmSearchAdmin/crmSearchAdminDisplay.component.js @@ -65,21 +65,6 @@ } }; - // Return all possible links to main entity or join entities - this.getLinks = function() { - var links = _.cloneDeep(searchMeta.getEntity(ctrl.savedSearch.api_entity).paths || []); - _.each(ctrl.savedSearch.api_params.join, function(join) { - var joinName = join[0].split(' AS '), - joinEntity = searchMeta.getEntity(joinName[0]); - _.each(joinEntity.paths, function(path) { - var link = _.cloneDeep(path); - link.path = link.path.replace(/\[/g, '[' + joinName[1] + '.'); - links.push(link); - }); - }); - return links; - }; - this.preview = this.stale = false; this.previewDisplay = function() { diff --git a/ext/search/ang/crmSearchAdmin/crmSearchAdminLinkSelect.directive.js b/ext/search/ang/crmSearchAdmin/crmSearchAdminLinkSelect.component.js similarity index 60% rename from ext/search/ang/crmSearchAdmin/crmSearchAdminLinkSelect.directive.js rename to ext/search/ang/crmSearchAdmin/crmSearchAdminLinkSelect.component.js index 667605546479..1a63886985f6 100644 --- a/ext/search/ang/crmSearchAdmin/crmSearchAdminLinkSelect.directive.js +++ b/ext/search/ang/crmSearchAdmin/crmSearchAdminLinkSelect.component.js @@ -4,13 +4,29 @@ angular.module('crmSearchAdmin').component('crmSearchAdminLinkSelect', { bindings: { column: '<', - links: '<' + apiEntity: '<', + apiParams: '<' }, templateUrl: '~/crmSearchAdmin/crmSearchAdminLinkSelect.html', - controller: function ($scope, $element, $timeout) { + controller: function ($scope, $element, $timeout, searchMeta) { var ts = $scope.ts = CRM.ts(), ctrl = this; + // Return all possible links to main entity or join entities + function getLinks() { + var links = _.cloneDeep(searchMeta.getEntity(ctrl.apiEntity).paths || []); + _.each(ctrl.apiParams.join, function(join) { + var joinName = join[0].split(' AS '), + joinEntity = searchMeta.getEntity(joinName[0]); + _.each(joinEntity.paths, function(path) { + var link = _.cloneDeep(path); + link.path = link.path.replace(/\[/g, '[' + joinName[1] + '.'); + links.push(link); + }); + }); + return links; + } + function onChange() { var val = $('select', $element).val(); if (val !== ctrl.column.link) { @@ -31,6 +47,8 @@ } this.$onInit = function() { + this.links = getLinks(); + $('select', $element).on('change', function() { $scope.$apply(onChange); }); diff --git a/ext/search/ang/crmSearchAdmin/crmSearchAdminLinkSelect.html b/ext/search/ang/crmSearchAdmin/crmSearchAdminLinkSelect.html index 47648e7aa463..74bc72913017 100644 --- a/ext/search/ang/crmSearchAdmin/crmSearchAdminLinkSelect.html +++ b/ext/search/ang/crmSearchAdmin/crmSearchAdminLinkSelect.html @@ -7,4 +7,7 @@ {{ ts('Other...') }} - +
+ + +
diff --git a/ext/search/ang/crmSearchAdmin/crmSearchAdminTokenSelect.component.js b/ext/search/ang/crmSearchAdmin/crmSearchAdminTokenSelect.component.js new file mode 100644 index 000000000000..c2c4b2ea126a --- /dev/null +++ b/ext/search/ang/crmSearchAdmin/crmSearchAdminTokenSelect.component.js @@ -0,0 +1,51 @@ +(function(angular, $, _) { + "use strict"; + + angular.module('crmSearchAdmin').component('crmSearchAdminTokenSelect', { + bindings: { + apiEntity: '<', + apiParams: '<', + model: '<', + field: '@' + }, + templateUrl: '~/crmSearchAdmin/crmSearchAdminTokenSelect.html', + controller: function ($scope, $element, searchMeta) { + var ts = $scope.ts = CRM.ts(), + ctrl = this; + + this.initTokens = function() { + ctrl.tokens = ctrl.tokens || getTokens(); + }; + + this.insertToken = function(key) { + ctrl.model[ctrl.field] = (ctrl.model[ctrl.field] || '') + ctrl.tokens[key].token; + }; + + function getTokens() { + var tokens = { + id: { + token: '[id]', + label: searchMeta.getField('id', ctrl.apiEntity).label + } + }; + _.each(ctrl.apiParams.join, function(joinParams) { + var info = searchMeta.parseExpr(joinParams[0].split(' AS ')[1] + '.id'); + tokens[info.alias] = { + token: '[' + info.alias + ']', + label: info.field ? info.field.label : info.alias + }; + }); + _.each(ctrl.apiParams.select, function(expr) { + var info = searchMeta.parseExpr(expr); + tokens[info.alias] = { + token: '[' + info.alias + ']', + label: info.field ? info.field.label : info.alias + }; + }); + return tokens; + } + + } + }); + +})(angular, CRM.$, CRM._); diff --git a/ext/search/ang/crmSearchAdmin/crmSearchAdminTokenSelect.html b/ext/search/ang/crmSearchAdmin/crmSearchAdminTokenSelect.html new file mode 100644 index 000000000000..e30fc5d1e800 --- /dev/null +++ b/ext/search/ang/crmSearchAdmin/crmSearchAdminTokenSelect.html @@ -0,0 +1,10 @@ +
+ + +
diff --git a/ext/search/ang/crmSearchAdmin/displays/searchAdminDisplayList.component.js b/ext/search/ang/crmSearchAdmin/displays/searchAdminDisplayList.component.js index 8517f149b7eb..d3080f6ed851 100644 --- a/ext/search/ang/crmSearchAdmin/displays/searchAdminDisplayList.component.js +++ b/ext/search/ang/crmSearchAdmin/displays/searchAdminDisplayList.component.js @@ -55,7 +55,6 @@ }; } ctrl.hiddenColumns = ctrl.crmSearchAdminDisplay.initColumns(); - ctrl.links = ctrl.crmSearchAdminDisplay.getLinks(); }; } diff --git a/ext/search/ang/crmSearchAdmin/displays/searchAdminDisplayList.html b/ext/search/ang/crmSearchAdmin/displays/searchAdminDisplayList.html index de7e87eb75ed..e877777e81d3 100644 --- a/ext/search/ang/crmSearchAdmin/displays/searchAdminDisplayList.html +++ b/ext/search/ang/crmSearchAdmin/displays/searchAdminDisplayList.html @@ -45,11 +45,12 @@
- +
+
diff --git a/ext/search/ang/crmSearchAdmin/displays/searchAdminDisplayTable.component.js b/ext/search/ang/crmSearchAdmin/displays/searchAdminDisplayTable.component.js index 3ef0cd702a45..3dc937b70dc9 100644 --- a/ext/search/ang/crmSearchAdmin/displays/searchAdminDisplayTable.component.js +++ b/ext/search/ang/crmSearchAdmin/displays/searchAdminDisplayTable.component.js @@ -39,7 +39,6 @@ }; } ctrl.hiddenColumns = ctrl.crmSearchAdminDisplay.initColumns(); - ctrl.links = ctrl.crmSearchAdminDisplay.getLinks(); }; } diff --git a/ext/search/ang/crmSearchAdmin/displays/searchAdminDisplayTable.html b/ext/search/ang/crmSearchAdmin/displays/searchAdminDisplayTable.html index b6d5092ac5c3..3589b56c03f4 100644 --- a/ext/search/ang/crmSearchAdmin/displays/searchAdminDisplayTable.html +++ b/ext/search/ang/crmSearchAdmin/displays/searchAdminDisplayTable.html @@ -31,11 +31,12 @@
- +
+
diff --git a/ext/search/ang/crmSearchDisplay.module.js b/ext/search/ang/crmSearchDisplay.module.js index d9f5bf2152ac..8d9ed43d58a1 100644 --- a/ext/search/ang/crmSearchDisplay.module.js +++ b/ext/search/ang/crmSearchDisplay.module.js @@ -5,21 +5,25 @@ angular.module('crmSearchDisplay', CRM.angRequires('crmSearchDisplay')) .factory('searchDisplayUtils', function() { - function getUrl(link, row) { - var url = replaceTokens(link, row); - if (url.slice(0, 1) !== '/' && url.slice(0, 4) !== 'http') { - url = CRM.url(url); - } - return _.escape(url); - } function replaceTokens(str, data) { + if (!str) { + return ''; + } _.each(data, function(value, key) { str = str.replace('[' + key + ']', value); }); return str; } + function getUrl(link, row) { + var url = replaceTokens(link, row); + if (url.slice(0, 1) !== '/' && url.slice(0, 4) !== 'http') { + url = CRM.url(url); + } + return _.escape(url); + } + function formatSearchValue(row, col, value) { var type = col.dataType, result = value; @@ -99,7 +103,8 @@ formatSearchValue: formatSearchValue, canAggregate: canAggregate, prepareColumns: prepareColumns, - prepareParams: prepareParams + prepareParams: prepareParams, + replaceTokens: replaceTokens }; }); diff --git a/ext/search/ang/crmSearchDisplayList/crmSearchDisplayList.component.js b/ext/search/ang/crmSearchDisplayList/crmSearchDisplayList.component.js index caf250af116e..50157877bc72 100644 --- a/ext/search/ang/crmSearchDisplayList/crmSearchDisplayList.component.js +++ b/ext/search/ang/crmSearchDisplayList/crmSearchDisplayList.component.js @@ -18,6 +18,7 @@ this.apiParams = _.cloneDeep(this.apiParams); this.apiParams.limit = parseInt(this.settings.limit || 0, 10); this.columns = searchDisplayUtils.prepareColumns(this.settings.columns, this.apiParams); + $scope.displayUtils = searchDisplayUtils; }; this.getResults = function() { diff --git a/ext/search/ang/crmSearchDisplayList/crmSearchDisplayListItems.html b/ext/search/ang/crmSearchDisplayList/crmSearchDisplayListItems.html index 15b8b1674d1d..8a6ae4936716 100644 --- a/ext/search/ang/crmSearchDisplayList/crmSearchDisplayListItems.html +++ b/ext/search/ang/crmSearchDisplayList/crmSearchDisplayListItems.html @@ -1,4 +1,4 @@
  • -
    +
    • diff --git a/ext/search/ang/crmSearchDisplayTable/crmSearchDisplayTable.component.js b/ext/search/ang/crmSearchDisplayTable/crmSearchDisplayTable.component.js index 8ea65c7c9e61..127d5d9c1393 100644 --- a/ext/search/ang/crmSearchDisplayTable/crmSearchDisplayTable.component.js +++ b/ext/search/ang/crmSearchDisplayTable/crmSearchDisplayTable.component.js @@ -21,6 +21,7 @@ this.apiParams = _.cloneDeep(this.apiParams); this.apiParams.limit = parseInt(this.settings.limit || 0, 10); this.columns = searchDisplayUtils.prepareColumns(this.settings.columns, this.apiParams); + $scope.displayUtils = searchDisplayUtils; }; this.getResults = function() { diff --git a/ext/search/ang/crmSearchDisplayTable/crmSearchDisplayTable.html b/ext/search/ang/crmSearchDisplayTable/crmSearchDisplayTable.html index 59d55f2d5ff9..d23d5efe9c06 100644 --- a/ext/search/ang/crmSearchDisplayTable/crmSearchDisplayTable.html +++ b/ext/search/ang/crmSearchDisplayTable/crmSearchDisplayTable.html @@ -18,7 +18,7 @@ - + From ed18f50cd90857d34498638530bec48d4cb2c4c5 Mon Sep 17 00:00:00 2001 From: Coleman Watts Date: Tue, 22 Dec 2020 21:24:07 -0500 Subject: [PATCH 50/63] Search kit - fix bugs introduced during refactoring --- ext/search/ang/crmSearchDisplay.module.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/search/ang/crmSearchDisplay.module.js b/ext/search/ang/crmSearchDisplay.module.js index d9f5bf2152ac..f88a7d8f1222 100644 --- a/ext/search/ang/crmSearchDisplay.module.js +++ b/ext/search/ang/crmSearchDisplay.module.js @@ -25,7 +25,7 @@ result = value; if (_.isArray(value)) { return _.map(value, function(val) { - return formatSearchValue(col, val); + return formatSearchValue(row, col, val); }).join(', '); } if (value && (type === 'Date' || type === 'Timestamp') && /^\d{4}-\d{2}-\d{2}/.test(value)) { @@ -79,7 +79,7 @@ _.each(params.join, function(join) { var joinEntity = join[0].split(' AS ')[1], idField = joinEntity + '.id'; - if (!_.includes(params.select, idField) && !searchDisplayUtils.canAggregate('id', joinEntity + '.', params)) { + if (!_.includes(params.select, idField) && !canAggregate('id', joinEntity + '.', params)) { params.select.push(idField); } }); From 619f1c318e9b08203a90703a632c7a13f6824585 Mon Sep 17 00:00:00 2001 From: eileen Date: Wed, 23 Dec 2020 15:59:17 +1300 Subject: [PATCH 51/63] Remove code to retrieve premium data After a bit of digging I can't see an evidence we ever have premiums on the backoffice membership forms --- CRM/Member/Form/Membership.php | 1 - 1 file changed, 1 deletion(-) diff --git a/CRM/Member/Form/Membership.php b/CRM/Member/Form/Membership.php index 6ae16df52b70..667e535d2e3c 100644 --- a/CRM/Member/Form/Membership.php +++ b/CRM/Member/Form/Membership.php @@ -1901,7 +1901,6 @@ protected function processContribution( $result, $receiptDate, $recurringContributionID), $contributionParams ); - $contributionParams['non_deductible_amount'] = CRM_Contribute_Form_Contribution_Confirm::getNonDeductibleAmount($params, $financialType, FALSE, $form); $contributionParams['skipCleanMoney'] = TRUE; // @todo this is the wrong place for this - it should be done as close to form submission // as possible From b6d9eabc5ed7c6d548a17ebe41393cdf85304231 Mon Sep 17 00:00:00 2001 From: Andrew Hunt Date: Tue, 22 Dec 2020 23:35:19 -0500 Subject: [PATCH 52/63] 5.33.0 release notes: raw from script --- release-notes/5.33.0.md | 313 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 313 insertions(+) create mode 100644 release-notes/5.33.0.md diff --git a/release-notes/5.33.0.md b/release-notes/5.33.0.md new file mode 100644 index 000000000000..d0e6a9ab159f --- /dev/null +++ b/release-notes/5.33.0.md @@ -0,0 +1,313 @@ +# CiviCRM 5.33.0 + +Released January 6, 2021; + +- **[Features](#features)** +- **[Bugs resolved](#bugs)** +- **[Miscellany](#misc)** +- **[Credits](#credits)** + +## Features + +### Core CiviCRM + +- **crm- Missing Summary ([18994](https://github.com/civicrm/civicrm-core/pull/18994))** + +## Bugs resolved + +### Core CiviCRM + +- **Fix Invalid argument PHP warning ([19219](https://github.com/civicrm/civicrm-core/pull/19219))** + +- **DispatchPolicy - Actively report any upgrade problems with hook_civicrm_permission ([19217](https://github.com/civicrm/civicrm-core/pull/19217))** + +- **dev/core#2232 - Upgrade UI contaminates cache via l10n-js. Consolidate isUpgradeMode(). ([19192](https://github.com/civicrm/civicrm-core/pull/19192))** + +- **Add release-notes/5.32.2.md ([19195](https://github.com/civicrm/civicrm-core/pull/19195))** + +- **Fix failure to assign view tpl variables to view page if context=search is in the url ([19189](https://github.com/civicrm/civicrm-core/pull/19189))** + +- **dev/core#1019 Fix currency formatting of Total Amount on Event and Contribution pages (with multi-currency form support) ([19185](https://github.com/civicrm/civicrm-core/pull/19185))** + +- **dev/core#2248 Ensure variables are assigned to tpl for urls ([19183](https://github.com/civicrm/civicrm-core/pull/19183))** + +- **dev/core#2246 Fix failure to filter exports ([19176](https://github.com/civicrm/civicrm-core/pull/19176))** + +- **dev/core#2244 Simplify and consistently apply checking of whether financial acls are enabled ([19173](https://github.com/civicrm/civicrm-core/pull/19173))** + +- **APIv4 - Fix dynamic bridge joins (used by Search Kit) ([19159](https://github.com/civicrm/civicrm-core/pull/19159))** + +- **Add release-notes/5.32.1 ([19161](https://github.com/civicrm/civicrm-core/pull/19161))** + +- **Handle possibility of fee_amount = '' ([19120](https://github.com/civicrm/civicrm-core/pull/19120))** + +- **Search kit joins ([19150](https://github.com/civicrm/civicrm-core/pull/19150))** + +- **CiviEvent - Error registering participants via search task ([19125](https://github.com/civicrm/civicrm-core/pull/19125))** + +- **dev/core#2232 Permit hook_civicrm_container and some other prebootish… ([19141](https://github.com/civicrm/civicrm-core/pull/19141))** + +- **[NFC] dev/core#2235 Fix url link to the OrderAPI Documenation ([19139](https://github.com/civicrm/civicrm-core/pull/19139))** + +- **Fix mglaman mapping ([19134](https://github.com/civicrm/civicrm-core/pull/19134))** + +- **dev/core#2188 - Upgrader - Cleanup any invalid combinations of is_search_range ([19123](https://github.com/civicrm/civicrm-core/pull/19123))** + +- **dev/core#2231 fix failure to calculate next_scheduled_date ([19119](https://github.com/civicrm/civicrm-core/pull/19119))** + +- **Search Kit: Support robust joins in UI ([19105](https://github.com/civicrm/civicrm-core/pull/19105))** + +- **Unhide oauth extension ([19107](https://github.com/civicrm/civicrm-core/pull/19107))** + +- **Unable to export contacts in Civi 5.32+ ([19104](https://github.com/civicrm/civicrm-core/pull/19104))** + +- **Improve APIv4 metadata for RelationshipCache and Bridge entities in general ([19101](https://github.com/civicrm/civicrm-core/pull/19101))** + +- **5.32 ([19093](https://github.com/civicrm/civicrm-core/pull/19093))** + +- **5.32 ([19090](https://github.com/civicrm/civicrm-core/pull/19090))** + +- **dev/core#927 Add test demonstrating that an extraneous activity is being created & fix ([19014](https://github.com/civicrm/civicrm-core/pull/19014))** + +- **Afform - Update fields and HTML mode in web-based editors ([19053](https://github.com/civicrm/civicrm-core/pull/19053))** + +- **5.32 ([19087](https://github.com/civicrm/civicrm-core/pull/19087))** + +- **dev/core#1790: Add short delay before closing tooltip elements ([19082](https://github.com/civicrm/civicrm-core/pull/19082))** + +- **[NFC] Add in an Emoji test for APIv3 as per Eileen's PR ([19078](https://github.com/civicrm/civicrm-core/pull/19078))** + +- **Improve schema metadata for Search Kit ([19075](https://github.com/civicrm/civicrm-core/pull/19075))** + +- **5.32 ([19081](https://github.com/civicrm/civicrm-core/pull/19081))** + +- **5.32 ([19080](https://github.com/civicrm/civicrm-core/pull/19080))** + +- **dev/core#2165 Test for Handle emojis less fatally where not supported ([18918](https://github.com/civicrm/civicrm-core/pull/18918))** + +- **dev/drupal#149 Override sessionStart function for Drupal8 using appro… ([19044](https://github.com/civicrm/civicrm-core/pull/19044))** + +- **Protect against 404s when wpBasePage is mixed case ([19063](https://github.com/civicrm/civicrm-core/pull/19063))** + +- **Extract function that generates upgrade link for extensions ([19070](https://github.com/civicrm/civicrm-core/pull/19070))** + +- **REF Simplify tokenProcessor code ([18612](https://github.com/civicrm/civicrm-core/pull/18612))** + +- **Feature to provide mostly used countries in top section of Country select list ([19025](https://github.com/civicrm/civicrm-core/pull/19025))** + +- **Minor tabs cleanup toward fixing dev/core#2215 ([19065](https://github.com/civicrm/civicrm-core/pull/19065))** + +- **Remove use of nullArray in delete hooks ([19059](https://github.com/civicrm/civicrm-core/pull/19059))** + +- **Deprecate UFGroup::add parameter ids ([19060](https://github.com/civicrm/civicrm-core/pull/19060))** + +- **[REF] Remove excess handling around contact_id ([19050](https://github.com/civicrm/civicrm-core/pull/19050))** + +- **Remove extra handling around contact id ([19051](https://github.com/civicrm/civicrm-core/pull/19051))** + +- **Remove unused hook_civicrm_crudLink and switch to using metadata for crudLinks ([18916](https://github.com/civicrm/civicrm-core/pull/18916))** + +- **[REF] Cleanup Ang modules in core to follow conventions ([19052](https://github.com/civicrm/civicrm-core/pull/19052))** + +- **wordpress#63 Add action parameter to PCP shortcode ([19058](https://github.com/civicrm/civicrm-core/pull/19058))** + +- **Update Resource URL Helptext ([19046](https://github.com/civicrm/civicrm-core/pull/19046))** + +- **5.32 ([19056](https://github.com/civicrm/civicrm-core/pull/19056))** + +- **Delete outdated/unused crmExample Angular module ([19049](https://github.com/civicrm/civicrm-core/pull/19049))** + +- **[REF] Decouple crmD3 angular module from CiviMail ([19047](https://github.com/civicrm/civicrm-core/pull/19047))** + +- **[REF] Remove xssString as it is providing a false sense of security ([19045](https://github.com/civicrm/civicrm-core/pull/19045))** + +- **[NFC] Remove boilerplate comment from .ang.php files ([19048](https://github.com/civicrm/civicrm-core/pull/19048))** + +- **[REF] Simplify sendNotification determination ([19054](https://github.com/civicrm/civicrm-core/pull/19054))** + +- **dev/financial#148 fully deprecate validateData function ([19043](https://github.com/civicrm/civicrm-core/pull/19043))** + +- **Remove legacy check ([19042](https://github.com/civicrm/civicrm-core/pull/19042))** + +- **Afform - Generate dashlets based on `Afform.is_dashlet` property. ([19005](https://github.com/civicrm/civicrm-core/pull/19005))** + +- **[REF] remove obscure use of objects from A.net ([19040](https://github.com/civicrm/civicrm-core/pull/19040))** + +- **Improve bootstrap3 checkbox theming ([19006](https://github.com/civicrm/civicrm-core/pull/19006))** + +- **[REF] Stop passing objects to recur in paypal pro - pass specific objects ([19041](https://github.com/civicrm/civicrm-core/pull/19041))** + +- **[REF] remove obscure use of objects from Anet.ipn ([19039](https://github.com/civicrm/civicrm-core/pull/19039))** + +- **[REF] Remove code Coleman hates ([19038](https://github.com/civicrm/civicrm-core/pull/19038))** + +- **Remove some deprecated code ([19037](https://github.com/civicrm/civicrm-core/pull/19037))** + +- **[REF] Simplify single function to receive contribution not objects ([19032](https://github.com/civicrm/civicrm-core/pull/19032))** + +- **dev/financial#148 fold call to loadObjects ([19033](https://github.com/civicrm/civicrm-core/pull/19033))** + +- **dev/financial#148 fold call to loadObjects - a.net ([19035](https://github.com/civicrm/civicrm-core/pull/19035))** + +- **dev/financial#148 duplicate out call to validateObjects ([19034](https://github.com/civicrm/civicrm-core/pull/19034))** + +- **5.32 to master ([19036](https://github.com/civicrm/civicrm-core/pull/19036))** + +- **5.32 ([19030](https://github.com/civicrm/civicrm-core/pull/19030))** + +- **Event Full: fix translation regression ([19027](https://github.com/civicrm/civicrm-core/pull/19027))** + +- **dev/core#927 Update ContributionCancelActions to also handle 'failed' ([19015](https://github.com/civicrm/civicrm-core/pull/19015))** + +- **Add Grant v4 api ([19020](https://github.com/civicrm/civicrm-core/pull/19020))** + +- **5.32 ([19022](https://github.com/civicrm/civicrm-core/pull/19022))** + +- **Fix issue #2162: allow reports to filter multi-select fields and find entities with multiple selections ([18978](https://github.com/civicrm/civicrm-core/pull/18978))** + +- **Check for membership type fee before applying tax ([19007](https://github.com/civicrm/civicrm-core/pull/19007))** + +- **dev/core#927 [REF] Further removal on unreachable code in transitionComponents ([19012](https://github.com/civicrm/civicrm-core/pull/19012))** + +- **dev/financial#152 [REF] Pass specific BAO into the recur function rather than the vague 'objects' ([19016](https://github.com/civicrm/civicrm-core/pull/19016))** + +- **Use trait instead of class for Entity Bridges; add OptionList trait ([19010](https://github.com/civicrm/civicrm-core/pull/19010))** + +- **Remove unused params, return params from processFail ([18998](https://github.com/civicrm/civicrm-core/pull/18998))** + +- **Remove unused parameters from cancel ([18997](https://github.com/civicrm/civicrm-core/pull/18997))** + +- **changes social media iframes/scripts to links, simplifies markup, adds email & bootstrap classes ([18880](https://github.com/civicrm/civicrm-core/pull/18880))** + +- **dev/core#2153 #REF Remove outdated updateCustomValues function ([18959](https://github.com/civicrm/civicrm-core/pull/18959))** + +- **Fix APIv4 test to assert an exception is thrown ([19009](https://github.com/civicrm/civicrm-core/pull/19009))** + +- **[NFC] Minor margin tidy up ([19013](https://github.com/civicrm/civicrm-core/pull/19013))** + +- **Add standard Contact fields to the Bookkeeping report template ([19008](https://github.com/civicrm/civicrm-core/pull/19008))** + +- **E_NOTICE when deleting participant ([19011](https://github.com/civicrm/civicrm-core/pull/19011))** + +- **5.32 ([19004](https://github.com/civicrm/civicrm-core/pull/19004))** + +- **Allow Angular modules to require Resource bundles ([18987](https://github.com/civicrm/civicrm-core/pull/18987))** + +- **5.32 to master ([19003](https://github.com/civicrm/civicrm-core/pull/19003))** + +- **5.32 ([19000](https://github.com/civicrm/civicrm-core/pull/19000))** + +- **dev/core#2066 Extract getSelectedIDs ([18772](https://github.com/civicrm/civicrm-core/pull/18772))** + +- **[REF] Remove always FALSE variable from transitionComponents ([18983](https://github.com/civicrm/civicrm-core/pull/18983))** + +- **dev/drupal#146 - Wrong link for Drupal 8 permissions page / Make CMS permissions url lookup more OO-ey ([18986](https://github.com/civicrm/civicrm-core/pull/18986))** + +- **dev/core#1931 Prevent PayPal from double-encoding the IPN Notify URL ([18980](https://github.com/civicrm/civicrm-core/pull/18980))** + +- **dev/financial#153 Fix redirect to PayPal ([18993](https://github.com/civicrm/civicrm-core/pull/18993))** + +- **dev/core#2197 Deploy monaco-editor using composer.json / composer dow… ([18988](https://github.com/civicrm/civicrm-core/pull/18988))** + +- **dev/core#2196 - serialize E_NOTICE when saving custom field ([18991](https://github.com/civicrm/civicrm-core/pull/18991))** + +- **afform - Get default field `