Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Export code cleanup: Convert relationships to pseudonames at the point of loading #12586

Merged
merged 2 commits into from
Oct 29, 2018
Merged

Export code cleanup: Convert relationships to pseudonames at the point of loading #12586

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ce12a9e
Convert relationships to pseudonames at the point of loading
eileenmcnaughton Jul 28, 2018
cedeaa5
Fix double handling of fields with unique names.
eileenmcnaughton Oct 25, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 79 additions & 60 deletions CRM/Export/BAO/Export.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -357,11 +357,13 @@ public static function exportComponents(
$returnProperties[$householdRelationshipType][$key] = $value;
}
}
// @todo - don't use returnProperties above.
$processor->setHouseholdMergeReturnProperties($returnProperties[$householdRelationshipType]);
}
}
}

list($relationQuery, $allRelContactArray) = self::buildRelatedContactArray($selectAll, $ids, $processor, $componentTable, $returnProperties);
self::buildRelatedContactArray($selectAll, $ids, $processor, $componentTable);

// make sure the groups stuff is included only if specifically specified
// by the fields param (CRM-1969), else we limit the contacts outputted to only
Expand Down Expand Up @@ -510,9 +512,20 @@ public static function exportComponents(
}

if ($processor->isRelationshipTypeKey($field)) {
$relDAO = CRM_Utils_Array::value($iterationDAO->contact_id, $allRelContactArray[$field]);
$relationQuery[$field]->convertToPseudoNames($relDAO);
self::fetchRelationshipDetails($relDAO, $value, $field, $row);
foreach (array_keys($value) as $property) {
if ($property === 'location') {
// @todo just undo all this nasty location wrangling!
foreach ($value['location'] as $locationKey => $locationFields) {
foreach (array_keys($locationFields) as $locationField) {
$fieldKey = str_replace(' ', '_', $locationKey . '-' . $locationField);
$row[$field . '_' . $fieldKey] = $processor->getRelationshipValue($field, $iterationDAO->contact_id, $fieldKey);
}
}
}
else {
$row[$field . '_' . $property] = $processor->getRelationshipValue($field, $iterationDAO->contact_id, $property);
}
}
}
else {
$row[$field] = self::getTransformedFieldValue($field, $iterationDAO, $fieldValue, $i18n, $metadata, $paymentDetails, $processor);
Expand Down Expand Up @@ -1110,16 +1123,11 @@ public static function _buildMasterCopyArray($sql, $exportParams, $sharedAddress
*/
public static function mergeSameHousehold($exportTempTable, &$sqlColumns, $prefix) {
$prefixColumn = $prefix . '_';
$allKeys = array_keys($sqlColumns);
$replaced = array();

// name map of the non standard fields in header rows & sql columns
$mappingFields = array(
'civicrm_primary_id' => 'id',
'contact_source' => 'source',
'current_employer_id' => 'employer_id',
'contact_is_deleted' => 'is_deleted',
'name' => 'address_name',
'provider_id' => 'im_service_provider',
'phone_type_id' => 'phone_type',
);
Expand Down Expand Up @@ -1400,7 +1408,7 @@ public static function getExportStructureArrays($returnProperties, $processor) {
$headerName = $field . '-' . 'current_employer';
}
else {
$headerName = $field . '-' . $queryFields[$relationField]['name'];
$headerName = $field . '-' . $relationField;
}
}

Expand Down Expand Up @@ -1502,6 +1510,8 @@ private static function fetchRelationshipDetails($relDAO, $value, $field, &$row)
$phoneTypes = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Phone', 'phone_type_id');
$imProviders = CRM_Core_PseudoConstant::get('CRM_Core_DAO_IM', 'provider_id');
$i18n = CRM_Core_I18n::singleton();
$field = $field . '_';

foreach ($value as $relationField => $relationValue) {
if (is_object($relDAO) && property_exists($relDAO, $relationField)) {
$fieldValue = $relDAO->$relationField;
Expand Down Expand Up @@ -1532,7 +1542,6 @@ private static function fetchRelationshipDetails($relDAO, $value, $field, &$row)
else {
$fieldValue = '';
}
$field = $field . '_';
$relPrefix = $field . $relationField;

if (is_object($relDAO) && $relationField == 'id') {
Expand Down Expand Up @@ -1649,64 +1658,74 @@ protected static function getIDsForRelatedContact($ids, $exportMode) {
* @param $ids
* @param \CRM_Export_BAO_ExportProcessor $processor
* @param $componentTable
* @param $returnProperties
*
* @return array
*/
protected static function buildRelatedContactArray($selectAll, $ids, $processor, $componentTable, $returnProperties) {
protected static function buildRelatedContactArray($selectAll, $ids, $processor, $componentTable) {
$allRelContactArray = $relationQuery = array();
$queryMode = $processor->getQueryMode();
$exportMode = $processor->getExportMode();
foreach (self::$relationshipTypes as $rel => $dnt) {
if ($relationReturnProperties = CRM_Utils_Array::value($rel, $returnProperties)) {
$allRelContactArray[$rel] = array();
// build Query for each relationship
$relationQuery[$rel] = new CRM_Contact_BAO_Query(NULL, $relationReturnProperties,
NULL, FALSE, FALSE, $queryMode
);
list($relationSelect, $relationFrom, $relationWhere, $relationHaving) = $relationQuery[$rel]->query();

list($id, $direction) = explode('_', $rel, 2);
// identify the relationship direction
$contactA = 'contact_id_a';
$contactB = 'contact_id_b';
if ($direction == 'b_a') {
$contactA = 'contact_id_b';
$contactB = 'contact_id_a';
}
$relIDs = self::getIDsForRelatedContact($ids, $exportMode);

$relationshipJoin = $relationshipClause = '';
if (!$selectAll && $componentTable) {
$relationshipJoin = " INNER JOIN {$componentTable} ctTable ON ctTable.contact_id = {$contactA}";
}
elseif (!empty($relIDs)) {
$relID = implode(',', $relIDs);
$relationshipClause = " AND crel.{$contactA} IN ( {$relID} )";
}
foreach ($processor->getRelationshipReturnProperties() as $relationshipKey => $relationReturnProperties) {
$allRelContactArray[$relationshipKey] = array();
// build Query for each relationship
$relationQuery = new CRM_Contact_BAO_Query(NULL, $relationReturnProperties,
NULL, FALSE, FALSE, $queryMode
);
list($relationSelect, $relationFrom, $relationWhere, $relationHaving) = $relationQuery->query();

list($id, $direction) = explode('_', $relationshipKey, 2);
// identify the relationship direction
$contactA = 'contact_id_a';
$contactB = 'contact_id_b';
if ($direction == 'b_a') {
$contactA = 'contact_id_b';
$contactB = 'contact_id_a';
}
$relIDs = self::getIDsForRelatedContact($ids, $exportMode);

$relationshipJoin = $relationshipClause = '';
if (!$selectAll && $componentTable) {
$relationshipJoin = " INNER JOIN {$componentTable} ctTable ON ctTable.contact_id = {$contactA}";
}
elseif (!empty($relIDs)) {
$relID = implode(',', $relIDs);
$relationshipClause = " AND crel.{$contactA} IN ( {$relID} )";
}

$relationFrom = " {$relationFrom}
INNER JOIN civicrm_relationship crel ON crel.{$contactB} = contact_a.id AND crel.relationship_type_id = {$id}
{$relationshipJoin} ";

//check for active relationship status only
$today = date('Ymd');
$relationActive = " AND (crel.is_active = 1 AND ( crel.end_date is NULL OR crel.end_date >= {$today} ) )";
$relationWhere = " WHERE contact_a.is_deleted = 0 {$relationshipClause} {$relationActive}";
$relationGroupBy = CRM_Contact_BAO_Query::getGroupByFromSelectColumns($relationQuery[$rel]->_select, "crel.{$contactA}");
$relationSelect = "{$relationSelect}, {$contactA} as refContact ";
$relationQueryString = "$relationSelect $relationFrom $relationWhere $relationHaving $relationGroupBy";

$allRelContactDAO = CRM_Core_DAO::executeQuery($relationQueryString);
while ($allRelContactDAO->fetch()) {
//FIX Me: Migrate this to table rather than array
// build the array of all related contacts
$allRelContactArray[$rel][$allRelContactDAO->refContact] = clone($allRelContactDAO);
$relationFrom = " {$relationFrom}
INNER JOIN civicrm_relationship crel ON crel.{$contactB} = contact_a.id AND crel.relationship_type_id = {$id}
{$relationshipJoin} ";

//check for active relationship status only
$today = date('Ymd');
$relationActive = " AND (crel.is_active = 1 AND ( crel.end_date is NULL OR crel.end_date >= {$today} ) )";
$relationWhere = " WHERE contact_a.is_deleted = 0 {$relationshipClause} {$relationActive}";
$relationGroupBy = CRM_Contact_BAO_Query::getGroupByFromSelectColumns($relationQuery->_select, "crel.{$contactA}");
$relationSelect = "{$relationSelect}, {$contactA} as refContact ";
$relationQueryString = "$relationSelect $relationFrom $relationWhere $relationHaving $relationGroupBy";

$allRelContactDAO = CRM_Core_DAO::executeQuery($relationQueryString);
while ($allRelContactDAO->fetch()) {
$relationQuery->convertToPseudoNames($allRelContactDAO);
$row = [];
// @todo pass processor to fetchRelationshipDetails and set fields directly within it.
self::fetchRelationshipDetails($allRelContactDAO, $relationReturnProperties, $relationshipKey, $row);
foreach (array_keys($relationReturnProperties) as $property) {
if ($property === 'location') {
// @todo - simplify location in self::fetchRelationshipDetails - remove handling here. Or just call
// $processor->setRelationshipValue from fetchRelationshipDetails
foreach ($relationReturnProperties['location'] as $locationName => $locationValues) {
foreach (array_keys($locationValues) as $locationValue) {
$key = str_replace(' ', '_', $locationName) . '-' . $locationValue;
$processor->setRelationshipValue($relationshipKey, $allRelContactDAO->refContact, $key, $row[$relationshipKey . '__' . $key]);
}
}
}
else {
$processor->setRelationshipValue($relationshipKey, $allRelContactDAO->refContact, $property, $row[$relationshipKey . '_' . $property]);
}
}
$allRelContactDAO->free();
}
}
return array($relationQuery, $allRelContactArray);
}

/**
Expand Down
65 changes: 64 additions & 1 deletion CRM/Export/BAO/ExportProcessor.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,21 @@ class CRM_Export_BAO_ExportProcessor {
*/
protected $relationshipReturnProperties = [];

/**
* Get return properties by relationship.
* @return array
*/
public function getRelationshipReturnProperties() {
return $this->relationshipReturnProperties;
}

/**
* Export values for related contacts.
*
* @var array
*/
protected $relatedContactValues = [];

/**
* @var array
*/
Expand Down Expand Up @@ -131,7 +146,6 @@ public function setRequestedFields($requestedFields) {
$this->requestedFields = $requestedFields;
}


/**
* @return array
*/
Expand Down Expand Up @@ -167,6 +181,41 @@ public function setRelationshipTypes() {
);
}

/**
* Set the value for a relationship type field.
*
* In this case we are building up an array of properties for a related contact.
*
* These may be used for direct exporting or for merge to household depending on the
* options selected.
*
* @param string $relationshipType
* @param int $contactID
* @param string $field
* @param string $value
*/
public function setRelationshipValue($relationshipType, $contactID, $field, $value) {
$this->relatedContactValues[$relationshipType][$contactID][$field] = $value;
}

/**
* Get the value for a relationship type field.
*
* In this case we are building up an array of properties for a related contact.
*
* These may be used for direct exporting or for merge to household depending on the
* options selected.
*
* @param string $relationshipType
* @param int $contactID
* @param string $field
*
* @return string
*/
public function getRelationshipValue($relationshipType, $contactID, $field) {
return isset($this->relatedContactValues[$relationshipType][$contactID][$field]) ? $this->relatedContactValues[$relationshipType][$contactID][$field] : '';
}

/**
* @return bool
*/
Expand Down Expand Up @@ -567,6 +616,20 @@ public function setRelationshipReturnProperties($value, $relationshipKey) {
return $this->relationshipReturnProperties[$relationshipKey];
}

/**
* Add the main return properties to the household merge properties if needed for merging.
*
* If we are using household merge we need to add these to the relationship properties to
* be retrieved.
*
* @param $returnProperties
*/
public function setHouseholdMergeReturnProperties($returnProperties) {
foreach ($this->getHouseholdRelationshipTypes() as $householdRelationshipType) {
$this->relationshipReturnProperties[$householdRelationshipType] = $returnProperties;
}
}

/**
* Get the default location fields to request.
*
Expand Down
52 changes: 51 additions & 1 deletion tests/phpunit/CRM/Export/BAO/ExportTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -419,6 +419,51 @@ public function testExportPseudoFieldCampaign() {
$this->assertEquals('Big campaign,', CRM_Core_DAO::singleValueQuery("SELECT GROUP_CONCAT(contribution_campaign_title) FROM {$tableName}"));
}

/**
* Test exporting relationships.
*/
public function testExportRelationships() {
$organization1 = $this->organizationCreate(['organization_name' => 'Org 1', 'legal_name' => 'pretty legal', 'contact_source' => 'friend who took a law paper once']);
$organization2 = $this->organizationCreate(['organization_name' => 'Org 2', 'legal_name' => 'well dodgey']);
$contact1 = $this->individualCreate(['employer_id' => $organization1, 'first_name' => 'one']);
$contact2 = $this->individualCreate(['employer_id' => $organization2, 'first_name' => 'one']);
$employerRelationshipTypeID = $this->callAPISuccessGetValue('RelationshipType', ['return' => 'id', 'label_a_b' => 'Employee of']);
$selectedFields = [
['Individual', 'first_name', ''],
['Individual', $employerRelationshipTypeID . '_a_b', 'organization_name', ''],
['Individual', $employerRelationshipTypeID . '_a_b', 'legal_name', ''],
['Individual', $employerRelationshipTypeID . '_a_b', 'contact_source', ''],
];
list($tableName, $sqlColumns, $headerRows) = CRM_Export_BAO_Export::exportComponents(
FALSE,
[$contact1, $contact2],
[],
NULL,
$selectedFields,
NULL,
CRM_Export_Form_Select::CONTACT_EXPORT,
"contact_a.id IN ( $contact1, $contact2 )",
NULL,
FALSE,
FALSE,
[
'exportOption' => CRM_Export_Form_Select::CONTACT_EXPORT,
'suppress_csv_for_testing' => TRUE,
]
);

$dao = CRM_Core_DAO::executeQuery("SELECT * FROM {$tableName}");
$dao->fetch();
$this->assertEquals('one', $dao->first_name);
$this->assertEquals('Org 1', $dao->{$employerRelationshipTypeID . '_a_b_organization_name'});
$this->assertEquals('pretty legal', $dao->{$employerRelationshipTypeID . '_a_b_legal_name'});
$this->assertEquals('friend who took a law paper once', $dao->{$employerRelationshipTypeID . '_a_b_contact_source'});

$dao->fetch();
$this->assertEquals('Org 2', $dao->{$employerRelationshipTypeID . '_a_b_organization_name'});
$this->assertEquals('well dodgey', $dao->{$employerRelationshipTypeID . '_a_b_legal_name'});
}

/**
* Test exporting relationships.
*
Expand All @@ -432,6 +477,7 @@ public function testExportRelationshipsMergeToHousehold() {
['Individual', $houseHoldTypeID . '_a_b', 'city', ''],
['Individual', 'city', ''],
['Individual', 'state_province', ''],
['Individual', 'contact_source', ''],
];
list($tableName, $sqlColumns, $headerRows) = CRM_Export_BAO_Export::exportComponents(
FALSE,
Expand All @@ -456,18 +502,21 @@ public function testExportRelationshipsMergeToHousehold() {
$this->assertEquals('ME', $dao->state_province);
$this->assertEquals($householdID, $dao->civicrm_primary_id);
$this->assertEquals($householdID, $dao->civicrm_primary_id);
$this->assertEquals('household sauce', $dao->contact_source);
}

$this->assertEquals([
0 => 'City',
1 => 'State',
2 => 'Household ID',
2 => 'Contact Source',
3 => 'Household ID',
], $headerRows);
$this->assertEquals(
[
'city' => 'city varchar(64)',
'state_province' => 'state_province varchar(64)',
'civicrm_primary_id' => 'civicrm_primary_id varchar(16)',
'contact_source' => 'contact_source varchar(255)',
], $sqlColumns);
}

Expand Down Expand Up @@ -956,6 +1005,7 @@ public function testExportDeceasedDoNotMail() {
protected function setUpHousehold() {
$this->setUpContactExportData();
$householdID = $this->householdCreate([
'source' => 'household sauce',
'api.Address.create' => [
'city' => 'Portland',
'state_province_id' => 'Maine',
Expand Down