From 8b3f43f48bdc91c7ed07237f2d8ef1033ffe24b2 Mon Sep 17 00:00:00 2001
From: Eileen McNaughton <emcnaughton@wikimedia.org>
Date: Fri, 10 Jun 2022 20:32:46 +1200
Subject: [PATCH] =?UTF-8?q?dev/core#3505,=20dev/core#3506=20Import=20fixes?=
 =?UTF-8?q?=20for=20unicode=20url,=20C=C3=B4te=20d=E2=80=99Ivoire?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 CRM/Import/Parser.php                         | 24 +++++++++++++++----
 CRM/Utils/Rule.php                            |  2 +-
 .../Import/Form/data/individual_unicode.csv   |  2 ++
 .../CRM/Contact/Import/Parser/ContactTest.php |  6 +++++
 4 files changed, 29 insertions(+), 5 deletions(-)
 create mode 100644 tests/phpunit/CRM/Contact/Import/Form/data/individual_unicode.csv

diff --git a/CRM/Import/Parser.php b/CRM/Import/Parser.php
index 7ec35b7c9aa7..753f0a5077d5 100644
--- a/CRM/Import/Parser.php
+++ b/CRM/Import/Parser.php
@@ -1364,7 +1364,7 @@ protected function getTransformedFieldValue(string $fieldName, $importedValue) {
         return $importedValue;
       }
 
-      $comparisonValue = is_numeric($importedValue) ? $importedValue : mb_strtolower($importedValue);
+      $comparisonValue = $this->getComparisonValue($importedValue);
       return $options[$comparisonValue] ?? 'invalid_import_value';
     }
     if (!empty($fieldMetadata['FKClassName']) || !empty($fieldMetadata['pseudoconstant']['prefetch'])) {
@@ -1468,10 +1468,10 @@ protected function getFieldMetadata(string $fieldName, bool $loadOptions = FALSE
         // name AND label as either might be used. We also lower case before checking
         $values = [];
         foreach ($options as $option) {
-          $idKey = is_numeric($option['id']) ? $option['id'] : mb_strtolower($option['id']);
+          $idKey = $this->getComparisonValue($option['id']);
           $values[$idKey] = $option['id'];
           foreach (['name', 'label', 'abbr'] as $key) {
-            $optionValue = mb_strtolower($option[$key] ?? '');
+            $optionValue = $this->getComparisonValue($option[$key] ?? '');
             if ($optionValue !== '') {
               if (isset($values[$optionValue]) && $values[$optionValue] !== $option['id']) {
                 if (!isset($this->ambiguousOptions[$fieldName][$optionValue])) {
@@ -1775,7 +1775,7 @@ protected function getSiteDefaultCountry(): int {
    * @param string $importedValue
    */
   protected function isAmbiguous(string $fieldName, $importedValue): bool {
-    return !empty($this->ambiguousOptions[$fieldName][mb_strtolower($importedValue)]);
+    return !empty($this->ambiguousOptions[$fieldName][$this->getComparisonValue($importedValue)]);
   }
 
   /**
@@ -2020,4 +2020,20 @@ public static function formatCustomDate(&$params, &$formatted, $dateType, $dateP
     $formatted[$dateParam] = CRM_Utils_Date::processDate($params[$dateParam]);
   }
 
+  /**
+   * Get the value to use for option comparison purposes.
+   *
+   * We do a case-insensitive comparison, also swapping ’ for '
+   * which has at least one known usage (Côte d’Ivoire).
+   *
+   * Note we do this to both sides of the comparison.
+   *
+   * @param int|string|false|null $importedValue
+   *
+   * @return false|int|string|null
+   */
+  protected function getComparisonValue($importedValue) {
+    return is_numeric($importedValue) ? $importedValue : mb_strtolower(str_replace('’', "'", $importedValue));
+  }
+
 }
diff --git a/CRM/Utils/Rule.php b/CRM/Utils/Rule.php
index 4ba685588749..611f2281043a 100644
--- a/CRM/Utils/Rule.php
+++ b/CRM/Utils/Rule.php
@@ -221,7 +221,7 @@ public static function url($url) {
       // allow relative URL's (CRM-15598)
       $url = 'http://' . $_SERVER['HTTP_HOST'] . $url;
     }
-    return (bool) filter_var($url, FILTER_VALIDATE_URL);
+    return (bool) filter_var(self::idnToAsci($url), FILTER_VALIDATE_URL);
   }
 
   /**
diff --git a/tests/phpunit/CRM/Contact/Import/Form/data/individual_unicode.csv b/tests/phpunit/CRM/Contact/Import/Form/data/individual_unicode.csv
new file mode 100644
index 000000000000..576915bdf768
--- /dev/null
+++ b/tests/phpunit/CRM/Contact/Import/Form/data/individual_unicode.csv
@@ -0,0 +1,2 @@
+first_name,Last Name,Website,Country
+Iron,Man,https://কানাডা.com,Côte d'Ivoire
diff --git a/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php b/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php
index 00c907789639..5f70b8b6e4c8 100644
--- a/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php
+++ b/tests/phpunit/CRM/Contact/Import/Parser/ContactTest.php
@@ -1075,6 +1075,12 @@ public function testImport($csv, $mapper, $expectedError, $expectedOutcomes = []
    */
   public function importDataProvider(): array {
     return [
+      'individual_unicode.csv' => [
+        'csv' => 'individual_unicode.csv',
+        'mapper' => [['first_name'], ['last_name'], ['url', 1], ['country', 1]],
+        'expected_error' => '',
+        'expected_outcomes' => [CRM_Import_Parser::VALID => 1],
+      ],
       'individual_invalid_sub_type' => [
         'csv' => 'individual_invalid_contact_sub_type.csv',
         'mapper' => [['first_name'], ['last_name'], ['contact_sub_type']],