Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

dev/core#3498 Fix mishandled option values #23736

Merged
merged 3 commits into from
Jun 9, 2022
Merged

dev/core#3498 Fix mishandled option values #23736

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
86b783d
dev/core#3498 Fix mishandled option values
eileenmcnaughton Jun 9, 2022
332fe30
Extra fix to headerPatterns so fields 'guess' better
eileenmcnaughton Jun 9, 2022
5a9b0b7
Add missing test file
eileenmcnaughton Jun 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 25 additions & 20 deletions CRM/Custom/Import/Parser/Api.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<?php

use Civi\Api4\CustomGroup;
use Civi\Api4\CustomField;

/**
* Class CRM_Custom_Import_Parser_Api
Expand Down Expand Up @@ -81,8 +81,8 @@ public function setFieldMetadata(): void {
$importableFields = $this->getGroupFieldsForImport($customGroupID);
$this->importableFieldsMetadata = array_merge([
'do_not_import' => ['title' => ts('- do not import -')],
'contact_id' => ['title' => ts('Contact ID'), 'name' => 'contact_id', 'type' => CRM_Utils_Type::T_INT, 'options' => FALSE],
'external_identifier' => ['title' => ts('External Identifier'), 'name' => 'external_identifier', 'type' => CRM_Utils_Type::T_INT, 'options' => FALSE],
'contact_id' => ['title' => ts('Contact ID'), 'name' => 'contact_id', 'type' => CRM_Utils_Type::T_INT, 'options' => FALSE, 'headerPattern' => '/contact?|id$/i'],
'external_identifier' => ['title' => ts('External Identifier'), 'name' => 'external_identifier', 'type' => CRM_Utils_Type::T_INT, 'options' => FALSE, 'headerPattern' => '/external\s?id/i'],
], $importableFields);
}
}
Expand Down Expand Up @@ -175,35 +175,40 @@ private function formatCommonData($params, &$formatted) {
*
* @return array
*
* @noinspection PhpDocMissingThrowsInspection
* @noinspection PhpUnhandledExceptionInspection
*/
private function getGroupFieldsForImport($customGroupID) {
private function getGroupFieldsForImport(int $customGroupID): array {
$importableFields = [];
$params = ['custom_group_id' => $customGroupID];
$group = CustomGroup::get(FALSE)->addSelect('extends')->addWhere('id', '=', $customGroupID)->execute()->first();
$allFields = civicrm_api3('custom_field', 'get', $params);
$fields = $allFields['values'];
foreach ($fields as $id => $values) {
$fields = (array) CustomField::get(FALSE)
->addSelect('*', 'custom_group_id.is_multiple', 'custom_group_id.name', 'custom_group_id.extends')
->addWhere('custom_group_id', '=', $customGroupID)->execute();

foreach ($fields as $values) {
$datatype = $values['data_type'] ?? NULL;
if ($datatype === 'File') {
continue;
}
/* generate the key for the fields array */
$key = "custom_$id";
$regexp = preg_replace('/[.,;:!?]/', '', CRM_Utils_Array::value(0, $values));
$key = 'custom_' . $values['id'];
$regexp = preg_replace('/[.,;:!?]/', '', $values['label']);
$importableFields[$key] = [
'name' => $key,
'title' => $values['label'] ?? NULL,
'headerPattern' => '/' . preg_quote($regexp, '/') . '/',
'headerPattern' => '/' . preg_quote($regexp, '/') . '/i',
'import' => 1,
'custom_field_id' => $id,
'options_per_line' => $values['options_per_line'] ?? NULL,
'data_type' => $values['data_type'] ?? NULL,
'html_type' => $values['html_type'] ?? NULL,
'custom_field_id' => $values['id'],
'options_per_line' => $values['options_per_line'],
'data_type' => $values['data_type'],
'html_type' => $values['html_type'],
'type' => CRM_Core_BAO_CustomField::dataToType()[$values['data_type']],
'is_search_range' => $values['is_search_range'] ?? NULL,
'date_format' => $values['date_format'] ?? NULL,
'time_format' => $values['time_format'] ?? NULL,
'extends' => $group['extends'],
'is_search_range' => $values['is_search_range'],
'date_format' => $values['date_format'],
'time_format' => $values['time_format'],
'extends' => $values['custom_group_id.extends'],
'custom_group_id' => $customGroupID,
'custom_group_id.name' => $values['custom_group_id.name'],
'is_multiple' => $values['custom_group_id.is_multiple'],
];
}
return $importableFields;
Expand Down
41 changes: 25 additions & 16 deletions CRM/Import/Parser.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -571,9 +571,9 @@ public function getSelectTypes() {
*/
public function getHeaderPatterns(): array {
$values = [];
foreach ($this->_fields as $name => $field) {
if (isset($field->_headerPattern)) {
$values[$name] = $field->_headerPattern;
foreach ($this->importableFieldsMetadata as $name => $field) {
if (isset($field['headerPattern'])) {
$values[$name] = $field['headerPattern'] ?: '//';
}
}
return $values;
Expand Down Expand Up @@ -1441,19 +1441,28 @@ protected function getFieldMetadata(string $fieldName, bool $loadOptions = FALSE
}
$optionFieldName = empty($fieldMap[$fieldName]) ? $fieldMetadata['name'] : $fieldName;

if (!empty($fieldMetadata['custom_group_id'])) {
$customField = CustomField::get(FALSE)
->addWhere('id', '=', $fieldMetadata['custom_field_id'])
->addSelect('name', 'custom_group_id.name')
->execute()
->first();
$optionFieldName = $customField['custom_group_id.name'] . '.' . $customField['name'];
}
$options = civicrm_api4($this->getFieldEntity($fieldName), 'getFields', [
'loadOptions' => ['id', 'name', 'label', 'abbr'],
'where' => [['name', '=', $optionFieldName]],
'select' => ['options'],
])->first()['options'];
if (!empty($fieldMetadata['custom_field_id']) && !empty($fieldMetadata['is_multiple'])) {
$options = civicrm_api4('Custom_' . $fieldMetadata['custom_group_id.name'], 'getFields', [
'loadOptions' => ['id', 'name', 'label', 'abbr'],
'where' => [['custom_field_id', '=', $fieldMetadata['custom_field_id']]],
'select' => ['options'],
])->first()['options'];
}
else {
if (!empty($fieldMetadata['custom_group_id'])) {
$customField = CustomField::get(FALSE)
->addWhere('id', '=', $fieldMetadata['custom_field_id'])
->addSelect('name', 'custom_group_id.name')
->execute()
->first();
$optionFieldName = $customField['custom_group_id.name'] . '.' . $customField['name'];
}
$options = civicrm_api4($this->getFieldEntity($fieldName), 'getFields', [
'loadOptions' => ['id', 'name', 'label', 'abbr'],
'where' => [['name', '=', $optionFieldName]],
'select' => ['options'],
])->first()['options'];
}
if (is_array($options)) {
// We create an array of the possible variants - notably including
// name AND label as either might be used. We also lower case before checking
Expand Down
59 changes: 41 additions & 18 deletions tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,6 @@ class CRM_Contribute_Import_Parser_ContributionTest extends CiviUnitTestCase {
*/
protected $entity = 'Contribution';

/**
* @var int
*/
protected $userJobID;

/**
* Cleanup function.
*
Expand Down Expand Up @@ -363,21 +358,49 @@ protected function addRandomOption(string $optionGroup = 'payment_instrument'):
}

/**
* @param array $mappings
* Get the import's datasource form.
*
* @return array
* Defaults to contribution - other classes should override.
*
* @param array $submittedValues
*
* @return \CRM_Contribute_Import_Form_DataSource|\CRM_Core_Form|\CRM_Custom_Import_Form_DataSource
* @noinspection PhpUnnecessaryLocalVariableInspection
*/
protected function getMapperFromFieldMappings(array $mappings): array {
$mapper = [];
foreach ($mappings as $mapping) {
$fieldInput = [$mapping['name']];
if (!empty($mapping['soft_credit_type_id'])) {
$fieldInput[1] = $mapping['soft_credit_match_field'];
$fieldInput[2] = $mapping['soft_credit_type_id'];
}
$mapper[] = $fieldInput;
}
return $mapper;
protected function getDataSourceForm(array $submittedValues) {
return $this->getFormObject('CRM_Contribute_Import_Form_DataSource', $submittedValues);
}

/**
* Get the import's mapField form.
*
* Defaults to contribution - other classes should override.
*
* @param array $submittedValues
*
* @return \CRM_Contribute_Import_Form_MapField
* @noinspection PhpUnnecessaryLocalVariableInspection
*/
protected function getMapFieldForm(array $submittedValues): CRM_Contribute_Import_Form_MapField {
/* @var \CRM_Contribute_Import_Form_MapField $form */
$form = $this->getFormObject('CRM_Contribute_Import_Form_MapField', $submittedValues);
return $form;
}

/**
* Get the import's preview form.
*
* Defaults to contribution - other classes should override.
*
* @param array $submittedValues
*
* @return \CRM_Contribute_Import_Form_Preview
* @noinspection PhpUnnecessaryLocalVariableInspection
*/
protected function getPreviewForm(array $submittedValues): CRM_Contribute_Import_Form_Preview {
/* @var CRM_Contribute_Import_Form_Preview $form */
$form = $this->getFormObject('CRM_Contribute_Import_Form_Preview', $submittedValues);
return $form;
}

}
89 changes: 89 additions & 0 deletions tests/phpunit/CRM/Custom/Import/Parser/ApiTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
<?php
/**
* @file
* File for the CRM_Custom_Import_Parser_ContributionTest class.
*/

/**
* Test Contribution import parser.
*
* @package CiviCRM
* @group headless
*/
class CRM_Custom_Import_Parser_ApiTest extends CiviUnitTestCase {

use CRMTraits_Custom_CustomDataTrait;
use CRMTraits_Import_ParserTrait;

/**
* Test the full form-flow import.
*/
public function testImport(): void {
$this->individualCreate();
$this->createCustomGroupWithFieldOfType(['is_multiple' => TRUE, 'extends' => 'Contact'], 'select', 'level');
$customGroupID = $this->ids['CustomGroup']['level'];
$dateFieldID = $this->createDateCustomField(['date_format' => 'yy', 'custom_group_id' => $customGroupID])['id'];
$this->importCSV('custom_data_date_select.csv', [
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There's a test failure:

fopen(/home/jenkins/bknix-dfl/build/core-23736-3prig/web/sites/all/modules/civicrm/tests/phpunit/CRM/Custom/Import/Parser/data/custom_data_date_select.csv): Failed to open stream: No such file or directory

I think maybe this file wasn't committed?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice change from the extra file commits @demeritcowboy has been culling for me :-)

['name' => 'contact_id'],
['name' => $this->getCustomFieldName('levelselect')],
['name' => 'do_not_import'],
['name' => 'custom_' . $dateFieldID],
], ['multipleCustomData' => $customGroupID]);
$dataSource = new CRM_Import_DataSource_CSV($this->userJobID);
$row = $dataSource->getRow();
$this->assertEquals('IMPORTED', $row['_status']);
$row = $dataSource->getRow();
$this->assertEquals('IMPORTED', $row['_status']);
$row = $dataSource->getRow();
$this->assertEquals('ERROR', $row['_status']);
}

/**
* Get the import's datasource form.
*
* Defaults to contribution - other classes should override.
*
* @param array $submittedValues
*
* @return \CRM_Custom_Import_Form_DataSource
* @noinspection PhpUnnecessaryLocalVariableInspection
*/
protected function getDataSourceForm(array $submittedValues): CRM_Custom_Import_Form_DataSource {
/* @var \CRM_Custom_Import_Form_DataSource $form */
$form = $this->getFormObject('CRM_Custom_Import_Form_DataSource', $submittedValues);
return $form;
}

/**
* Get the import's mapField form.
*
* Defaults to contribution - other classes should override.
*
* @param array $submittedValues
*
* @return \CRM_Custom_Import_Form_MapField
* @noinspection PhpUnnecessaryLocalVariableInspection
*/
protected function getMapFieldForm(array $submittedValues): CRM_Custom_Import_Form_MapField {
/* @var \CRM_Custom_Import_Form_MapField $form */
$form = $this->getFormObject('CRM_Custom_Import_Form_MapField', $submittedValues);
return $form;
}

/**
* Get the import's preview form.
*
* Defaults to contribution - other classes should override.
*
* @param array $submittedValues
*
* @return \CRM_Custom_Import_Form_Preview
* @noinspection PhpUnnecessaryLocalVariableInspection
*/
protected function getPreviewForm(array $submittedValues): CRM_Custom_Import_Form_Preview {
/* @var CRM_Custom_Import_Form_Preview $form */
$form = $this->getFormObject('CRM_Custom_Import_Form_Preview', $submittedValues);
return $form;
}

}
4 changes: 4 additions & 0 deletions tests/phpunit/CRM/Custom/Import/Parser/data/custom_data_date_select.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
The Contact ID,The Level,The School,The Start Year
3,Red,Ah Ha,2010-12-01
3,R,Bah bah,2022-01-02
3,Aqua,Cah cah,2022-02-03
65 changes: 22 additions & 43 deletions tests/phpunit/CRMTraits/Import/ParserTrait.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@
*/
trait CRMTraits_Import_ParserTrait {

/**
* @var int
*/
protected $userJobID;

/**
* Import the csv file values.
*
Expand All @@ -41,9 +46,13 @@ protected function importCSV(string $csv, array $fieldMappings, array $submitted
'groups' => [],
], $submittedValues);
$form = $this->getDataSourceForm($submittedValues);
$values = $_SESSION['_' . $form->controller->_name . '_container']['values'];
$form->buildForm();
$form->postProcess();
$this->userJobID = $form->getUserJobID();
// This gets reset in DataSource so re-do....
$_SESSION['_' . $form->controller->_name . '_container']['values'] = $values;

$form = $this->getMapFieldForm($submittedValues);
$form->setUserJobID($this->userJobID);
$form->buildForm();
Expand All @@ -57,51 +66,21 @@ protected function importCSV(string $csv, array $fieldMappings, array $submitted
}

/**
* Get the import's datasource form.
*
* Defaults to contribution - other classes should override.
*
* @param array $submittedValues
*
* @return \CRM_Contribute_Import_Form_DataSource
* @noinspection PhpUnnecessaryLocalVariableInspection
*/
protected function getDataSourceForm(array $submittedValues): CRM_Contribute_Import_Form_DataSource {
/* @var \CRM_Contribute_Import_Form_DataSource $form */
$form = $this->getFormObject('CRM_Contribute_Import_Form_DataSource', $submittedValues);
return $form;
}

/**
* Get the import's mapField form.
*
* Defaults to contribution - other classes should override.
*
* @param array $submittedValues
*
* @return \CRM_Contribute_Import_Form_MapField
* @noinspection PhpUnnecessaryLocalVariableInspection
*/
protected function getMapFieldForm(array $submittedValues): CRM_Contribute_Import_Form_MapField {
/* @var \CRM_Contribute_Import_Form_MapField $form */
$form = $this->getFormObject('CRM_Contribute_Import_Form_MapField', $submittedValues);
return $form;
}

/**
* Get the import's preview form.
*
* Defaults to contribution - other classes should override.
*
* @param array $submittedValues
* @param array $mappings
*
* @return \CRM_Contribute_Import_Form_Preview
* @noinspection PhpUnnecessaryLocalVariableInspection
* @return array
*/
protected function getPreviewForm(array $submittedValues): CRM_Contribute_Import_Form_Preview {
/* @var CRM_Contribute_Import_Form_Preview $form */
$form = $this->getFormObject('CRM_Contribute_Import_Form_Preview', $submittedValues);
return $form;
protected function getMapperFromFieldMappings(array $mappings): array {
$mapper = [];
foreach ($mappings as $mapping) {
$fieldInput = [$mapping['name']];
if (!empty($mapping['soft_credit_type_id'])) {
$fieldInput[1] = $mapping['soft_credit_match_field'];
$fieldInput[2] = $mapping['soft_credit_type_id'];
}
$mapper[] = $fieldInput;
}
return $mapper;
}

}
Loading