From ce12a9e0288b5b8cc6cb4cc95331509a6133c060 Mon Sep 17 00:00:00 2001 From: eileen Date: Sat, 28 Jul 2018 16:38:28 +1200 Subject: [PATCH 1/2] Convert relationships to pseudonames at the point of loading --- CRM/Export/BAO/Export.php | 132 ++++++++++++-------- CRM/Export/BAO/ExportProcessor.php | 65 +++++++++- tests/phpunit/CRM/Export/BAO/ExportTest.php | 43 +++++++ 3 files changed, 185 insertions(+), 55 deletions(-) diff --git a/CRM/Export/BAO/Export.php b/CRM/Export/BAO/Export.php index fec624e51fd6..3f7f906aa0b8 100644 --- a/CRM/Export/BAO/Export.php +++ b/CRM/Export/BAO/Export.php @@ -357,11 +357,13 @@ public static function exportComponents( $returnProperties[$householdRelationshipType][$key] = $value; } } + // @todo - don't use returnProperties above. + $processor->setHouseholdMergeReturnProperties($returnProperties[$householdRelationshipType]); } } } - list($relationQuery, $allRelContactArray) = self::buildRelatedContactArray($selectAll, $ids, $processor, $componentTable, $returnProperties); + self::buildRelatedContactArray($selectAll, $ids, $processor, $componentTable); // make sure the groups stuff is included only if specifically specified // by the fields param (CRM-1969), else we limit the contacts outputted to only @@ -510,9 +512,20 @@ public static function exportComponents( } if ($processor->isRelationshipTypeKey($field)) { - $relDAO = CRM_Utils_Array::value($iterationDAO->contact_id, $allRelContactArray[$field]); - $relationQuery[$field]->convertToPseudoNames($relDAO); - self::fetchRelationshipDetails($relDAO, $value, $field, $row); + foreach (array_keys($value) as $property) { + if ($property === 'location') { + // @todo just undo all this nasty location wrangling! + foreach ($value['location'] as $locationKey => $locationFields) { + foreach (array_keys($locationFields) as $locationField) { + $fieldKey = str_replace(' ', '_', $locationKey . '-' . $locationField); + $row[$field . '_' . $fieldKey] = $processor->getRelationshipValue($field, $iterationDAO->contact_id, $fieldKey); + } + } + } + else { + $row[$field . '_' . $property] = $processor->getRelationshipValue($field, $iterationDAO->contact_id, $property); + } + } } else { $row[$field] = self::getTransformedFieldValue($field, $iterationDAO, $fieldValue, $i18n, $metadata, $paymentDetails, $processor); @@ -1502,6 +1515,8 @@ private static function fetchRelationshipDetails($relDAO, $value, $field, &$row) $phoneTypes = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Phone', 'phone_type_id'); $imProviders = CRM_Core_PseudoConstant::get('CRM_Core_DAO_IM', 'provider_id'); $i18n = CRM_Core_I18n::singleton(); + $field = $field . '_'; + foreach ($value as $relationField => $relationValue) { if (is_object($relDAO) && property_exists($relDAO, $relationField)) { $fieldValue = $relDAO->$relationField; @@ -1532,7 +1547,6 @@ private static function fetchRelationshipDetails($relDAO, $value, $field, &$row) else { $fieldValue = ''; } - $field = $field . '_'; $relPrefix = $field . $relationField; if (is_object($relDAO) && $relationField == 'id') { @@ -1649,64 +1663,74 @@ protected static function getIDsForRelatedContact($ids, $exportMode) { * @param $ids * @param \CRM_Export_BAO_ExportProcessor $processor * @param $componentTable - * @param $returnProperties - * - * @return array */ - protected static function buildRelatedContactArray($selectAll, $ids, $processor, $componentTable, $returnProperties) { + protected static function buildRelatedContactArray($selectAll, $ids, $processor, $componentTable) { $allRelContactArray = $relationQuery = array(); $queryMode = $processor->getQueryMode(); $exportMode = $processor->getExportMode(); - foreach (self::$relationshipTypes as $rel => $dnt) { - if ($relationReturnProperties = CRM_Utils_Array::value($rel, $returnProperties)) { - $allRelContactArray[$rel] = array(); - // build Query for each relationship - $relationQuery[$rel] = new CRM_Contact_BAO_Query(NULL, $relationReturnProperties, - NULL, FALSE, FALSE, $queryMode - ); - list($relationSelect, $relationFrom, $relationWhere, $relationHaving) = $relationQuery[$rel]->query(); - - list($id, $direction) = explode('_', $rel, 2); - // identify the relationship direction - $contactA = 'contact_id_a'; - $contactB = 'contact_id_b'; - if ($direction == 'b_a') { - $contactA = 'contact_id_b'; - $contactB = 'contact_id_a'; - } - $relIDs = self::getIDsForRelatedContact($ids, $exportMode); - $relationshipJoin = $relationshipClause = ''; - if (!$selectAll && $componentTable) { - $relationshipJoin = " INNER JOIN {$componentTable} ctTable ON ctTable.contact_id = {$contactA}"; - } - elseif (!empty($relIDs)) { - $relID = implode(',', $relIDs); - $relationshipClause = " AND crel.{$contactA} IN ( {$relID} )"; - } + foreach ($processor->getRelationshipReturnProperties() as $relationshipKey => $relationReturnProperties) { + $allRelContactArray[$relationshipKey] = array(); + // build Query for each relationship + $relationQuery = new CRM_Contact_BAO_Query(NULL, $relationReturnProperties, + NULL, FALSE, FALSE, $queryMode + ); + list($relationSelect, $relationFrom, $relationWhere, $relationHaving) = $relationQuery->query(); + + list($id, $direction) = explode('_', $relationshipKey, 2); + // identify the relationship direction + $contactA = 'contact_id_a'; + $contactB = 'contact_id_b'; + if ($direction == 'b_a') { + $contactA = 'contact_id_b'; + $contactB = 'contact_id_a'; + } + $relIDs = self::getIDsForRelatedContact($ids, $exportMode); + + $relationshipJoin = $relationshipClause = ''; + if (!$selectAll && $componentTable) { + $relationshipJoin = " INNER JOIN {$componentTable} ctTable ON ctTable.contact_id = {$contactA}"; + } + elseif (!empty($relIDs)) { + $relID = implode(',', $relIDs); + $relationshipClause = " AND crel.{$contactA} IN ( {$relID} )"; + } - $relationFrom = " {$relationFrom} - INNER JOIN civicrm_relationship crel ON crel.{$contactB} = contact_a.id AND crel.relationship_type_id = {$id} - {$relationshipJoin} "; - - //check for active relationship status only - $today = date('Ymd'); - $relationActive = " AND (crel.is_active = 1 AND ( crel.end_date is NULL OR crel.end_date >= {$today} ) )"; - $relationWhere = " WHERE contact_a.is_deleted = 0 {$relationshipClause} {$relationActive}"; - $relationGroupBy = CRM_Contact_BAO_Query::getGroupByFromSelectColumns($relationQuery[$rel]->_select, "crel.{$contactA}"); - $relationSelect = "{$relationSelect}, {$contactA} as refContact "; - $relationQueryString = "$relationSelect $relationFrom $relationWhere $relationHaving $relationGroupBy"; - - $allRelContactDAO = CRM_Core_DAO::executeQuery($relationQueryString); - while ($allRelContactDAO->fetch()) { - //FIX Me: Migrate this to table rather than array - // build the array of all related contacts - $allRelContactArray[$rel][$allRelContactDAO->refContact] = clone($allRelContactDAO); + $relationFrom = " {$relationFrom} + INNER JOIN civicrm_relationship crel ON crel.{$contactB} = contact_a.id AND crel.relationship_type_id = {$id} + {$relationshipJoin} "; + + //check for active relationship status only + $today = date('Ymd'); + $relationActive = " AND (crel.is_active = 1 AND ( crel.end_date is NULL OR crel.end_date >= {$today} ) )"; + $relationWhere = " WHERE contact_a.is_deleted = 0 {$relationshipClause} {$relationActive}"; + $relationGroupBy = CRM_Contact_BAO_Query::getGroupByFromSelectColumns($relationQuery->_select, "crel.{$contactA}"); + $relationSelect = "{$relationSelect}, {$contactA} as refContact "; + $relationQueryString = "$relationSelect $relationFrom $relationWhere $relationHaving $relationGroupBy"; + + $allRelContactDAO = CRM_Core_DAO::executeQuery($relationQueryString); + while ($allRelContactDAO->fetch()) { + $relationQuery->convertToPseudoNames($allRelContactDAO); + $row = []; + // @todo pass processor to fetchRelationshipDetails and set fields directly within it. + self::fetchRelationshipDetails($allRelContactDAO, $relationReturnProperties, $relationshipKey, $row); + foreach (array_keys($relationReturnProperties) as $property) { + if ($property === 'location') { + // @todo - simplify location in self::fetchRelationshipDetails - remove handling here. Or just call + // $processor->setRelationshipValue from fetchRelationshipDetails + foreach ($relationReturnProperties['location'] as $locationName => $locationValues) { + foreach (array_keys($locationValues) as $locationValue) { + $key = str_replace(' ', '_', $locationName) . '-' . $locationValue; + $processor->setRelationshipValue($relationshipKey, $allRelContactDAO->refContact, $key, $row[$relationshipKey . '__' . $key]); + } + } + } + else { + $processor->setRelationshipValue($relationshipKey, $allRelContactDAO->refContact, $property, $row[$relationshipKey . '_' . $property]); + } } - $allRelContactDAO->free(); } } - return array($relationQuery, $allRelContactArray); } /** diff --git a/CRM/Export/BAO/ExportProcessor.php b/CRM/Export/BAO/ExportProcessor.php index 026befc13373..0b77a52eda6e 100644 --- a/CRM/Export/BAO/ExportProcessor.php +++ b/CRM/Export/BAO/ExportProcessor.php @@ -95,6 +95,21 @@ class CRM_Export_BAO_ExportProcessor { */ protected $relationshipReturnProperties = []; + /** + * Get return properties by relationship. + * @return array + */ + public function getRelationshipReturnProperties() { + return $this->relationshipReturnProperties; + } + + /** + * Export values for related contacts. + * + * @var array + */ + protected $relatedContactValues = []; + /** * @var array */ @@ -131,7 +146,6 @@ public function setRequestedFields($requestedFields) { $this->requestedFields = $requestedFields; } - /** * @return array */ @@ -167,6 +181,41 @@ public function setRelationshipTypes() { ); } + /** + * Set the value for a relationship type field. + * + * In this case we are building up an array of properties for a related contact. + * + * These may be used for direct exporting or for merge to household depending on the + * options selected. + * + * @param string $relationshipType + * @param int $contactID + * @param string $field + * @param string $value + */ + public function setRelationshipValue($relationshipType, $contactID, $field, $value) { + $this->relatedContactValues[$relationshipType][$contactID][$field] = $value; + } + + /** + * Get the value for a relationship type field. + * + * In this case we are building up an array of properties for a related contact. + * + * These may be used for direct exporting or for merge to household depending on the + * options selected. + * + * @param string $relationshipType + * @param int $contactID + * @param string $field + * + * @return string + */ + public function getRelationshipValue($relationshipType, $contactID, $field) { + return isset($this->relatedContactValues[$relationshipType][$contactID][$field]) ? $this->relatedContactValues[$relationshipType][$contactID][$field] : ''; + } + /** * @return bool */ @@ -567,6 +616,20 @@ public function setRelationshipReturnProperties($value, $relationshipKey) { return $this->relationshipReturnProperties[$relationshipKey]; } + /** + * Add the main return properties to the household merge properties if needed for merging. + * + * If we are using household merge we need to add these to the relationship properties to + * be retrieved. + * + * @param $returnProperties + */ + public function setHouseholdMergeReturnProperties($returnProperties) { + foreach ($this->getHouseholdRelationshipTypes() as $householdRelationshipType) { + $this->relationshipReturnProperties[$householdRelationshipType] = $returnProperties; + } + } + /** * Get the default location fields to request. * diff --git a/tests/phpunit/CRM/Export/BAO/ExportTest.php b/tests/phpunit/CRM/Export/BAO/ExportTest.php index cea24c8dcac3..985c434d3c15 100644 --- a/tests/phpunit/CRM/Export/BAO/ExportTest.php +++ b/tests/phpunit/CRM/Export/BAO/ExportTest.php @@ -419,6 +419,49 @@ public function testExportPseudoFieldCampaign() { $this->assertEquals('Big campaign,', CRM_Core_DAO::singleValueQuery("SELECT GROUP_CONCAT(contribution_campaign_title) FROM {$tableName}")); } + /** + * Test exporting relationships. + */ + public function testExportRelationships() { + $organization1 = $this->organizationCreate(['organization_name' => 'Org 1', 'legal_name' => 'pretty legal']); + $organization2 = $this->organizationCreate(['organization_name' => 'Org 2', 'legal_name' => 'well dodgey']); + $contact1 = $this->individualCreate(['employer_id' => $organization1, 'first_name' => 'one']); + $contact2 = $this->individualCreate(['employer_id' => $organization2, 'first_name' => 'one']); + $employerRelationshipTypeID = $this->callAPISuccessGetValue('RelationshipType', ['return' => 'id', 'label_a_b' => 'Employee of']); + $selectedFields = [ + ['Individual', 'first_name', ''], + ['Individual', $employerRelationshipTypeID . '_a_b', 'organization_name', ''], + ['Individual', $employerRelationshipTypeID . '_a_b', 'legal_name', ''], + ]; + list($tableName, $sqlColumns, $headerRows) = CRM_Export_BAO_Export::exportComponents( + FALSE, + [$contact1, $contact2], + [], + NULL, + $selectedFields, + NULL, + CRM_Export_Form_Select::CONTACT_EXPORT, + "contact_a.id IN ( $contact1, $contact2 )", + NULL, + FALSE, + FALSE, + [ + 'exportOption' => CRM_Export_Form_Select::CONTACT_EXPORT, + 'suppress_csv_for_testing' => TRUE, + ] + ); + + $dao = CRM_Core_DAO::executeQuery("SELECT * FROM {$tableName}"); + $dao->fetch(); + $this->assertEquals('one', $dao->first_name); + $this->assertEquals('Org 1', $dao->{$employerRelationshipTypeID . '_a_b_organization_name'}); + $this->assertEquals('pretty legal', $dao->{$employerRelationshipTypeID . '_a_b_legal_name'}); + + $dao->fetch(); + $this->assertEquals('Org 2', $dao->{$employerRelationshipTypeID . '_a_b_organization_name'}); + $this->assertEquals('well dodgey', $dao->{$employerRelationshipTypeID . '_a_b_legal_name'}); + } + /** * Test exporting relationships. * From cedeaa5cda7f29930417bdddaf2a5dd17ad73793 Mon Sep 17 00:00:00 2001 From: eileen Date: Fri, 26 Oct 2018 00:34:43 +1300 Subject: [PATCH 2/2] Fix double handling of fields with unique names. The relationship array is being built with 'names' & the main array with 'unique names' which later need to be wrangled back into sync. Do relationships the same as main array --- CRM/Export/BAO/Export.php | 7 +------ tests/phpunit/CRM/Export/BAO/ExportTest.php | 11 +++++++++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CRM/Export/BAO/Export.php b/CRM/Export/BAO/Export.php index 3f7f906aa0b8..00214262afd4 100644 --- a/CRM/Export/BAO/Export.php +++ b/CRM/Export/BAO/Export.php @@ -1123,16 +1123,11 @@ public static function _buildMasterCopyArray($sql, $exportParams, $sharedAddress */ public static function mergeSameHousehold($exportTempTable, &$sqlColumns, $prefix) { $prefixColumn = $prefix . '_'; - $allKeys = array_keys($sqlColumns); $replaced = array(); // name map of the non standard fields in header rows & sql columns $mappingFields = array( 'civicrm_primary_id' => 'id', - 'contact_source' => 'source', - 'current_employer_id' => 'employer_id', - 'contact_is_deleted' => 'is_deleted', - 'name' => 'address_name', 'provider_id' => 'im_service_provider', 'phone_type_id' => 'phone_type', ); @@ -1413,7 +1408,7 @@ public static function getExportStructureArrays($returnProperties, $processor) { $headerName = $field . '-' . 'current_employer'; } else { - $headerName = $field . '-' . $queryFields[$relationField]['name']; + $headerName = $field . '-' . $relationField; } } diff --git a/tests/phpunit/CRM/Export/BAO/ExportTest.php b/tests/phpunit/CRM/Export/BAO/ExportTest.php index 985c434d3c15..ff368a31d79d 100644 --- a/tests/phpunit/CRM/Export/BAO/ExportTest.php +++ b/tests/phpunit/CRM/Export/BAO/ExportTest.php @@ -423,7 +423,7 @@ public function testExportPseudoFieldCampaign() { * Test exporting relationships. */ public function testExportRelationships() { - $organization1 = $this->organizationCreate(['organization_name' => 'Org 1', 'legal_name' => 'pretty legal']); + $organization1 = $this->organizationCreate(['organization_name' => 'Org 1', 'legal_name' => 'pretty legal', 'contact_source' => 'friend who took a law paper once']); $organization2 = $this->organizationCreate(['organization_name' => 'Org 2', 'legal_name' => 'well dodgey']); $contact1 = $this->individualCreate(['employer_id' => $organization1, 'first_name' => 'one']); $contact2 = $this->individualCreate(['employer_id' => $organization2, 'first_name' => 'one']); @@ -432,6 +432,7 @@ public function testExportRelationships() { ['Individual', 'first_name', ''], ['Individual', $employerRelationshipTypeID . '_a_b', 'organization_name', ''], ['Individual', $employerRelationshipTypeID . '_a_b', 'legal_name', ''], + ['Individual', $employerRelationshipTypeID . '_a_b', 'contact_source', ''], ]; list($tableName, $sqlColumns, $headerRows) = CRM_Export_BAO_Export::exportComponents( FALSE, @@ -456,6 +457,7 @@ public function testExportRelationships() { $this->assertEquals('one', $dao->first_name); $this->assertEquals('Org 1', $dao->{$employerRelationshipTypeID . '_a_b_organization_name'}); $this->assertEquals('pretty legal', $dao->{$employerRelationshipTypeID . '_a_b_legal_name'}); + $this->assertEquals('friend who took a law paper once', $dao->{$employerRelationshipTypeID . '_a_b_contact_source'}); $dao->fetch(); $this->assertEquals('Org 2', $dao->{$employerRelationshipTypeID . '_a_b_organization_name'}); @@ -475,6 +477,7 @@ public function testExportRelationshipsMergeToHousehold() { ['Individual', $houseHoldTypeID . '_a_b', 'city', ''], ['Individual', 'city', ''], ['Individual', 'state_province', ''], + ['Individual', 'contact_source', ''], ]; list($tableName, $sqlColumns, $headerRows) = CRM_Export_BAO_Export::exportComponents( FALSE, @@ -499,18 +502,21 @@ public function testExportRelationshipsMergeToHousehold() { $this->assertEquals('ME', $dao->state_province); $this->assertEquals($householdID, $dao->civicrm_primary_id); $this->assertEquals($householdID, $dao->civicrm_primary_id); + $this->assertEquals('household sauce', $dao->contact_source); } $this->assertEquals([ 0 => 'City', 1 => 'State', - 2 => 'Household ID', + 2 => 'Contact Source', + 3 => 'Household ID', ], $headerRows); $this->assertEquals( [ 'city' => 'city varchar(64)', 'state_province' => 'state_province varchar(64)', 'civicrm_primary_id' => 'civicrm_primary_id varchar(16)', + 'contact_source' => 'contact_source varchar(255)', ], $sqlColumns); } @@ -999,6 +1005,7 @@ public function testExportDeceasedDoNotMail() { protected function setUpHousehold() { $this->setUpContactExportData(); $householdID = $this->householdCreate([ + 'source' => 'household sauce', 'api.Address.create' => [ 'city' => 'Portland', 'state_province_id' => 'Maine',