Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Export code cleanup - only construct one metadata array with all types of metadata #13126

Merged
merged 4 commits into from
Nov 21, 2018
Merged

Export code cleanup - only construct one metadata array with all types of metadata #13126

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
af17bed
Make getHeaderRows more sensible
eileenmcnaughton Nov 18, 2018
28254dc
Move definition of header rows to ExportProcessor
eileenmcnaughton Nov 19, 2018
de6ff50
Stop setting header rows only to unset them
eileenmcnaughton Nov 19, 2018
ae5d4ca
Move construction of headers to ExportProcessor
eileenmcnaughton Nov 19, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 16 additions & 47 deletions CRM/Export/BAO/Export.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,7 @@ public static function exportComponents(
if (!array_key_exists($column, $returnProperties)) {
$returnProperties[$column] = 1;
$column = $column == 'id' ? 'civicrm_primary_id' : $column;
$processor->setColumnAsCalculationOnly($column);
$exportParams['merge_same_address']['temp_columns'][$column] = 1;
}
}
Expand Down Expand Up @@ -319,6 +320,7 @@ public static function exportComponents(
if (!array_key_exists($column, $returnProperties)) {
$returnProperties[$column] = 1;
$exportParams['postal_mailing_export']['temp_columns'][$column] = 1;
$processor->setColumnAsCalculationOnly($column);
}
}
}
Expand Down Expand Up @@ -439,7 +441,8 @@ public static function exportComponents(

$count = -1;

list($outputColumns, $headerRows, $sqlColumns, $metadata) = self::getExportStructureArrays($returnProperties, $processor);
list($outputColumns, $sqlColumns, $metadata) = self::getExportStructureArrays($returnProperties, $processor);
$headerRows = $processor->getHeaderRows();

// add payment headers if required
if ($addPaymentHeader && $processor->isExportPaymentFields()) {
Expand Down Expand Up @@ -492,12 +495,12 @@ public static function exportComponents(
if (isset($exportParams['postal_mailing_export']['postal_mailing_export']) &&
$exportParams['postal_mailing_export']['postal_mailing_export'] == 1
) {
self::postalMailingFormat($exportTempTable, $headerRows, $sqlColumns, $exportMode);
self::postalMailingFormat($exportTempTable, $sqlColumns, $exportMode);
}

// do merge same address and merge same household processing
if ($mergeSameAddress) {
self::mergeSameAddress($exportTempTable, $headerRows, $sqlColumns, $exportParams);
self::mergeSameAddress($exportTempTable, $sqlColumns, $exportParams);
}

// merge the records if they have corresponding households
Expand Down Expand Up @@ -725,11 +728,10 @@ public static function createTempTable($sqlColumns) {

/**
* @param string $tableName
* @param $headerRows
* @param $sqlColumns
* @param array $exportParams
*/
public static function mergeSameAddress($tableName, &$headerRows, &$sqlColumns, $exportParams) {
public static function mergeSameAddress($tableName, &$sqlColumns, $exportParams) {
// check if any records are present based on if they have used shared address feature,
// and not based on if city / state .. matches.
$sql = "
Expand Down Expand Up @@ -826,7 +828,7 @@ public static function mergeSameAddress($tableName, &$headerRows, &$sqlColumns,
$unsetKeys = array_keys($sqlColumns);
foreach ($unsetKeys as $headerKey => $sqlColKey) {
if (array_key_exists($sqlColKey, $exportParams['merge_same_address']['temp_columns'])) {
unset($sqlColumns[$sqlColKey], $headerRows[$headerKey]);
unset($sqlColumns[$sqlColKey]);
}
}
}
Expand Down Expand Up @@ -1131,11 +1133,10 @@ public static function writeCSVFromTable($exportTempTable, $headerRows, $sqlColu
* Exclude contacts who are deceased, have "Do not mail" privacy setting,
* or have no street address
* @param $exportTempTable
* @param $headerRows
* @param $sqlColumns
* @param $exportParams
*/
public static function postalMailingFormat($exportTempTable, &$headerRows, &$sqlColumns, $exportParams) {
public static function postalMailingFormat($exportTempTable, &$sqlColumns, $exportParams) {
$whereClause = array();

if (array_key_exists('is_deceased', $sqlColumns)) {
Expand Down Expand Up @@ -1178,7 +1179,7 @@ public static function postalMailingFormat($exportTempTable, &$headerRows, &$sql
$unsetKeys = array_keys($sqlColumns);
foreach ($unsetKeys as $headerKey => $sqlColKey) {
if (array_key_exists($sqlColKey, $exportParams['postal_mailing_export']['temp_columns'])) {
unset($sqlColumns[$sqlColKey], $headerRows[$headerKey]);
unset($sqlColumns[$sqlColKey]);
}
}
}
Expand Down Expand Up @@ -1206,38 +1207,6 @@ public static function componentPaymentFields() {
return $componentPaymentFields;
}

/**
* Set the definition for the header rows and sql columns based on the field to output.
*
* @param string $field
* @param array $headerRows
* @param \CRM_Export_BAO_ExportProcessor $processor
*
* @return array
*/
public static function setHeaderRows($field, $headerRows, $processor) {

$queryFields = $processor->getQueryFields();
if (substr($field, -11) == 'campaign_id') {
// @todo - set this correctly in the xml rather than here.
$headerRows[] = ts('Campaign ID');
}
elseif ($processor->isMergeSameHousehold() && $field === 'id') {
$headerRows[] = ts('Household ID');
}
elseif (isset($queryFields[$field]['title'])) {
$headerRows[] = $queryFields[$field]['title'];
}
elseif ($processor->isExportPaymentFields() && array_key_exists($field, $processor->getcomponentPaymentFields())) {
$headerRows[] = CRM_Utils_Array::value($field, $processor->getcomponentPaymentFields());
}
else {
$headerRows[] = $field;
}

return $headerRows;
}

/**
* Get the various arrays that we use to structure our output.
*
Expand Down Expand Up @@ -1267,14 +1236,14 @@ public static function setHeaderRows($field, $headerRows, $processor) {
* yet find a way to comment them for posterity.
*/
public static function getExportStructureArrays($returnProperties, $processor) {
$metadata = $headerRows = $outputColumns = $sqlColumns = array();
$metadata = $outputColumns = $sqlColumns = array();
$phoneTypes = CRM_Core_PseudoConstant::get('CRM_Core_DAO_Phone', 'phone_type_id');
$imProviders = CRM_Core_PseudoConstant::get('CRM_Core_DAO_IM', 'provider_id');
$queryFields = $processor->getQueryFields();
foreach ($returnProperties as $key => $value) {
if (($key != 'location' || !is_array($value)) && !$processor->isRelationshipTypeKey($key)) {
$outputColumns[$key] = $value;
$headerRows = self::setHeaderRows($key, $headerRows, $processor);
$processor->addOutputSpecification($key);
self::sqlColumnDefn($processor, $sqlColumns, $key);
}
elseif ($processor->isRelationshipTypeKey($key)) {
Expand All @@ -1285,7 +1254,7 @@ public static function getExportStructureArrays($returnProperties, $processor) {
if (isset($queryFields[$relationField]['title'])) {
if (!$processor->isHouseholdMergeRelationshipTypeKey($field)) {
// Do not add to header row if we are only generating for merge reasons.
$headerRows[] = $processor->getRelationshipTypes()[$key] . '-' . $queryFields[$relationField]['title'];
$processor->addOutputSpecification($relationField, $key);
}
self::sqlColumnDefn($processor, $sqlColumns, $field . '-' . $relationField);
}
Expand All @@ -1305,7 +1274,7 @@ public static function getExportStructureArrays($returnProperties, $processor) {
$hdr .= "-" . CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_IM', 'provider_id', $type[1]);
}
}
$headerRows[] = $processor->getRelationshipTypes()[$key] . $hdr;
$processor->addOutputSpecification($field, $key, $ltype, CRM_Utils_Array::value(1, $type));
self::sqlColumnDefn($processor, $sqlColumns, $field . '-' . $hdr);
}
}
Expand Down Expand Up @@ -1335,7 +1304,7 @@ public static function getExportStructureArrays($returnProperties, $processor) {
$metadata[$daoFieldName]['pseudoconstant']['var'] = 'imProviders';
}
self::sqlColumnDefn($processor, $sqlColumns, $outputFieldName);
$headerRows = self::setHeaderRows($outputFieldName, $headerRows, $processor);
$processor->addOutputSpecification($outputFieldName, NULL, $locationType, CRM_Utils_Array::value(1, $type));
self::sqlColumnDefn($processor, $sqlColumns, $outputFieldName);
if ($actualDBFieldName == 'country' || $actualDBFieldName == 'world_region') {
$metadata[$daoFieldName] = array('context' => 'country');
Expand All @@ -1348,7 +1317,7 @@ public static function getExportStructureArrays($returnProperties, $processor) {
}
}
}
return array($outputColumns, $headerRows, $sqlColumns, $metadata);
return array($outputColumns, $sqlColumns, $metadata);
}

/**
Expand Down
87 changes: 87 additions & 0 deletions CRM/Export/BAO/ExportProcessor.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,11 @@ public function getRelationshipReturnProperties() {
*/
protected $returnProperties = [];

/**
* @var array
*/
protected $outputSpecification = [];
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A bit picky, but could we have a basic comment here explaining what this array is for?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah - I'll add in a follow up


/**
* CRM_Export_BAO_ExportProcessor constructor.
*
Expand Down Expand Up @@ -389,6 +394,33 @@ public function getExportFileName() {
}
}

/**
* Get the label for the header row based on the field to output.
*
* @param string $field
*
* @return string
*/
public function getHeaderForRow($field) {
if (substr($field, -11) == 'campaign_id') {
// @todo - set this correctly in the xml rather than here.
// This will require a generalised handling cleanup
return ts('Campaign ID');
}
if ($this->isMergeSameHousehold() && $field === 'id') {
return ts('Household ID');
}
elseif (isset($this->getQueryFields()[$field]['title'])) {
return $this->getQueryFields()[$field]['title'];
}
elseif ($this->isExportPaymentFields() && array_key_exists($field, $this->getcomponentPaymentFields())) {
return CRM_Utils_Array::value($field, $this->getcomponentPaymentFields());
}
else {
return $field;
}
}

/**
* @param $params
* @param $order
Expand All @@ -409,6 +441,61 @@ public function runQuery($params, $order, $returnProperties) {
return array($query, $select, $from, $where, $having);
}

/**
* Add a row to the specification for how to output data.
*
* @param string $key
* @param string $relationshipType
* @param string $locationType
* @param int $entityTypeID phone_type_id or provider_id for phone or im fields.
*/
public function addOutputSpecification($key, $relationshipType = NULL, $locationType = NULL, $entityTypeID = NULL) {
$label = $this->getHeaderForRow($key);
$labelPrefix = $fieldPrefix = [];
if ($relationshipType) {
$labelPrefix[] = $this->getRelationshipTypes()[$relationshipType];
$fieldPrefix[] = $relationshipType;
}
if ($locationType) {
$labelPrefix[] = $fieldPrefix[] = $locationType;
}
if ($entityTypeID) {
if ($key === 'phone') {
$labelPrefix[] = $fieldPrefix[] = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_Phone', 'phone_type_id', $entityTypeID);
}
if ($key === 'im') {
$labelPrefix[] = $fieldPrefix[] = CRM_Core_PseudoConstant::getLabel('CRM_Core_BAO_IM', 'provider_id', $entityTypeID);
}
}
$index = ($fieldPrefix ? (implode('-', $fieldPrefix) . '-') : '') . $key;
$this->outputSpecification[$index]['header'] = ($labelPrefix ? (implode('-', $labelPrefix) . '-') : '') . $label;

}

/**
* Mark a column as only required for calculations.
*
* Do not include the row with headers.
*
* @param string $column
*/
public function setColumnAsCalculationOnly($column) {
$this->outputSpecification[$column]['do_not_output_to_csv'] = TRUE;
}

/**
* @return array
*/
public function getHeaderRows() {
$headerRows = [];
foreach ($this->outputSpecification as $key => $spec) {
if (empty($spec['do_not_output_to_csv'])) {
$headerRows[] = $spec['header'];
}
}
return $headerRows;
}

/**
* Build the row for output.
*
Expand Down