[REF] [Import] Parser cleanup

CRM/Contact/Import/Form/DataSource.php

@@ -222,9 +222,6 @@ public function postProcess() {
       CRM_Import_Parser::MODE_MAPFIELD
     );
 
-    // add all the necessary variables to the form
-    $parser->set($this);
-
   }
 
   /**

CRM/Contact/Import/Form/MapField.php

@@ -111,7 +111,6 @@ public function preProcess() {
     $this->_columnCount = $this->getNumberOfColumns();
     $this->_columnNames = $columnNames;
     $this->assign('columnNames', $this->getColumnHeaders());
-    //$this->_columnCount = $this->get( 'columnCount' );
     $this->assign('columnCount', $this->_columnCount);
     $this->_dataValues = array_values($this->getDataRows([], 2));
     $this->assign('dataValues', $this->_dataValues);
@@ -134,7 +133,6 @@ public function buildQuickForm() {
     $defaults = [];
     $mapperKeys = array_keys($this->_mapperFields);
     $hasColumnNames = !empty($this->_columnNames);
-    $hasLocationTypes = $this->get('fieldTypes');
 
     $this->_location_types = ['Primary' => ts('Primary')] + CRM_Core_PseudoConstant::get('CRM_Core_DAO_Address', 'location_type_id');
     $defaultLocationType = CRM_Core_BAO_LocationType::getDefault();
@@ -207,7 +205,7 @@ public function buildQuickForm() {
         $values = [];
         foreach ($relatedFields as $name => $field) {
           $values[$name] = $field['title'];
-          if (isset($hasLocationTypes[$name])) {
+          if ($this->isLocationTypeRequired($name)) {
             $sel3[$key][$name] = $this->_location_types;
           }
           elseif ($name === 'url') {
@@ -271,7 +269,7 @@ public function buildQuickForm() {
       }
       else {
         $options = NULL;
-        if (!empty($hasLocationTypes[$key])) {
+        if ($this->isLocationTypeRequired($key)) {
           $options = $this->_location_types;
         }
         elseif ($key === 'url') {
@@ -651,4 +649,18 @@ private function getHighlightedFields(): array {
     return $highlightedFields;
   }
 
+  /**
+   * Get an array of fields with TRUE or FALSE to reflect need for location type.
+   *
+   * e.g ['first_name' => FALSE, 'email' => TRUE, 'street_address' => TRUE']
+   *
+   * @return bool
+   */
+  private function isLocationTypeRequired($name): bool {
+    if (!isset(Civi::$statics[__CLASS__]['location_fields'])) {
+      Civi::$statics[__CLASS__]['location_fields'] = (new CRM_Contact_Import_Parser_Contact())->setUserJobID($this->getUserJobID())->getSelectTypes();
+    }
+    return (bool) (Civi::$statics[__CLASS__]['location_fields'][$name] ?? FALSE);
+  }
+
 }

CRM/Contact/Import/Form/Preview.php

@@ -34,8 +34,8 @@ class CRM_Contact_Import_Form_Preview extends CRM_Import_Form_Preview {
    * @throws \CRM_Core_Exception
    */
   public function preProcess() {
-    $columnNames = $this->get('columnNames');
-    $this->_disableUSPS = $this->get('disableUSPS');
+    $columnNames = $this->getColumnHeaders();
+    $this->_disableUSPS = $this->getSubmittedValue('disableUSPS');
 
     //assign column names
     $this->assign('columnNames', $columnNames);
@@ -168,9 +168,9 @@ public static function formRule($fields, $files, $self) {
   public function postProcess() {
 
     $importJobParams = array(
-      'doGeocodeAddress' => $this->controller->exportValue('DataSource', 'doGeocodeAddress'),
-      'invalidRowCount' => $this->get('invalidRowCount'),
-      'onDuplicate' => $this->get('onDuplicate'),
+      'doGeocodeAddress' => $this->getSubmittedValue('doGeocodeAddress'),
+      'invalidRowCount' => $this->getRowCount(CRM_Import_Parser::ERROR),
+      'onDuplicate' => $this->getSubmittedValue('onDuplicate'),
       'dedupe' => $this->getSubmittedValue('dedupe_rule_id'),
       'newGroupName' => $this->controller->exportValue($this->_name, 'newGroupName'),
       'newGroupDesc' => $this->controller->exportValue($this->_name, 'newGroupDesc'),
@@ -188,7 +188,7 @@ public function postProcess() {
       'primaryKeyName' => '_id',
       'statusFieldName' => '_status',
       'statusID' => $this->get('statusID'),
-      'totalRowCount' => $this->get('totalRowCount'),
+      'totalRowCount' => $this->getRowCount([]),
       'userJobID' => $this->getUserJobID(),
     );
 

CRM/Contact/Import/Form/Summary.php

@@ -29,7 +29,7 @@ class CRM_Contact_Import_Form_Summary extends CRM_Import_Form_Summary {
   public function preProcess() {
     // set the error message path to display
     $this->assign('errorFile', $this->get('errorFile'));
-    $onDuplicate = $this->get('onDuplicate');
+    $onDuplicate = $this->getSubmittedValue('onDuplicate');
     $this->assign('dupeError', FALSE);
 
     if ($onDuplicate == CRM_Import_Parser::DUPLICATE_UPDATE) {

CRM/Contact/Import/Parser/Contact.php

@@ -811,7 +811,13 @@ public function import($onDuplicate, &$values) {
       return CRM_Import_Parser::ERROR;
 
     }
-    // sleep(3);
+
+    if (empty($this->_unparsedStreetAddressContacts)) {
+      $this->setImportStatus((int) ($values[count($values) - 1]), 'IMPORTED', '', $contactID);
+      return CRM_Import_Parser::VALID;
+    }
+
+    // @todo - record unparsed address as 'imported' but the presence of a message is meaningful?
     return $this->processMessage($values, $statusFieldName, CRM_Import_Parser::VALID);
   }
 
@@ -2265,16 +2271,14 @@ public function deprecated_contact_check_params(
    * @param array $mapper
    * @param int $mode
    * @param int $statusID
-   * @param int $totalRowCount
    *
    * @return mixed
    * @throws \API_Exception|\CRM_Core_Exception
    */
   public function run(
     $mapper = [],
     $mode = self::MODE_PREVIEW,
-    $statusID = NULL,
-    $totalRowCount = NULL
+    $statusID = NULL
   ) {
 
     // TODO: Make the timeout actually work
@@ -2288,32 +2292,19 @@ public function run(
     $this->init();
 
     $this->_rowCount = 0;
-    $this->_invalidRowCount = $this->_validCount = 0;
     $this->_totalCount = 0;
 
-    $this->_errors = [];
-    $this->_warnings = [];
-    $this->_unparsedAddresses = [];
-
-    $this->_tableName = $tableName = $this->getUserJob()['metadata']['DataSource']['table_name'];
     $this->_primaryKeyName = '_id';
     $this->_statusFieldName = '_status';
 
-    if ($mode == self::MODE_MAPFIELD) {
-      $this->_rows = [];
-    }
-    else {
-      $this->_activeFieldCount = count($this->_activeFields);
-    }
-
     if ($statusID) {
       $this->progressImport($statusID);
       $startTimestamp = $currTimestamp = $prevTimestamp = time();
     }
-    // get the contents of the temp. import table
-    $query = "SELECT * FROM $tableName";
+    $dataSource = $this->getDataSourceObject();
+    $totalRowCount = $dataSource->getRowCount(['new']);
     if ($mode == self::MODE_IMPORT) {
-      $query .= " WHERE _status = 'NEW'";
+      $dataSource->setStatuses(['new']);
     }
     if ($this->_maxLinesToProcess > 0) {
       // Note this would only be the case in MapForm mode, where it is set to 100
@@ -2325,20 +2316,13 @@ public function run(
       // which is the number of columns a row might have.
       // However, the mapField class may no longer use activeFieldsCount for contact
       // to be continued....
-      $query .= ' LIMIT ' . $this->_maxLinesToProcess;
+      $dataSource->setLimit($this->_maxLinesToProcess);
     }
 
-    $result = CRM_Core_DAO::executeQuery($query);
-
-    while ($result->fetch()) {
-      $values = array_values($result->toArray());
+    while ($row = $dataSource->getRow()) {
+      $values = array_values($row);
       $this->_rowCount++;
 
-      /* trim whitespace around the values */
-      foreach ($values as $k => $v) {
-        $values[$k] = trim($v, " \t\r\n");
-      }
-
       $this->_totalCount++;
 
       if ($mode == self::MODE_MAPFIELD) {
@@ -2369,30 +2353,6 @@ public function run(
         $returnCode = self::ERROR;
       }
 
-      // note that a line could be valid but still produce a warning
-      if ($returnCode & self::VALID) {
-        $this->_validCount++;
-        if ($mode == self::MODE_MAPFIELD) {
-          $this->_rows[] = $values;
-          $this->_activeFieldCount = max($this->_activeFieldCount, count($values));
-        }
-      }
-
-      if ($returnCode & self::ERROR) {
-        $this->_invalidRowCount++;
-        array_unshift($values, $this->_rowCount);
-        $this->_errors[] = $values;
-      }
-
-      if ($returnCode & self::DUPLICATE) {
-        $this->_duplicateCount++;
-        array_unshift($values, $this->_rowCount);
-        $this->_duplicates[] = $values;
-        if ($onDuplicate != self::DUPLICATE_SKIP) {
-          $this->_validCount++;
-        }
-      }
-
       if ($returnCode & self::NO_MATCH) {
         $this->setImportStatus((int) $values[count($values) - 1], 'invalid_no_match', array_shift($values));
       }
@@ -2411,7 +2371,6 @@ public function run(
    *   Mapped array of values.
    */
   public function setActiveFields($fieldKeys) {
-    $this->_activeFieldCount = count($fieldKeys);
     foreach ($fieldKeys as $key) {
       if (empty($this->_fields[$key])) {
         $this->_activeFields[] = new CRM_Contact_Import_Field('', ts('- do not import -'));
@@ -2578,8 +2537,6 @@ public function addField(
    * @param int $mode
    */
   public function set($store, $mode = self::MODE_SUMMARY) {
-    // To be removed in https://github.com/civicrm/civicrm-core/pull/23281
-    $store->set('fieldTypes', $this->getSelectTypes());
   }
 
   /**
@@ -2629,29 +2586,21 @@ public static function exportCSV($fileName, $header, $data) {
   }
 
   /**
-   * Set the import status for the given record.
-   *
-   * If this is a sql import then the sql table will be used and the update
-   * will not happen as the relevant fields don't exist in the table - hence
-   * the checks that statusField & primary key are set.
+   * Update the status of the import row to reflect the processing outcome.
    *
    * @param int $id
    * @param string $status
    * @param string $message
+   * @param int|null $entityID
+   *   Optional created entity ID
+   * @param array $relatedEntityIDs
+   *   Optional array e.g ['related_contact' => 4]
+   *
+   * @throws \API_Exception
+   * @throws \CRM_Core_Exception
    */
-  public function setImportStatus(int $id, string $status, string $message): void {
-    if ($this->_statusFieldName && $this->_primaryKeyName) {
-      CRM_Core_DAO::executeQuery("
-        UPDATE $this->_tableName
-        SET $this->_statusFieldName = %1,
-          {$this->_statusFieldName}Msg = %2
-        WHERE  $this->_primaryKeyName = %3
-      ", [
-        1 => [$status, 'String'],
-        2 => [$message, 'String'],
-        3 => [$id, 'Integer'],
-      ]);
-    }
+  public function setImportStatus(int $id, string $status, string $message, ?int $entityID = NULL, array $relatedEntityIDs = []): void {
+    $this->getDataSourceObject()->updateStatus($id, $status, $message, $entityID, $relatedEntityIDs);
   }
 
   /**
@@ -3133,19 +3082,6 @@ public function getMappedRow(array $values): array {
     return $params;
   }
 
-  /**
-   * Is the job complete.
-   *
-   * This function transitionally accesses the table from the userJob
-   * directly - but the function should be moved to the dataSource class.
-   *
-   * @throws \API_Exception
-   */
-  public function isComplete() {
-    $tableName = $this->getUserJob()['metadata']['DataSource']['table_name'];
-    return (bool) CRM_Core_DAO::singleValueQuery("SELECT count(*) FROM $tableName WHERE _status = 'NEW' LIMIT 1");
-  }
-
   /**
    * Validate the import values.
    *

CRM/Import/DataSource.php

@@ -442,12 +442,55 @@ public function purge(array $newParams = []) :array {
   protected function addTrackingFieldsToTable(string $tableName): void {
     CRM_Core_DAO::executeQuery("
      ALTER TABLE $tableName
+       ADD COLUMN _entity_id INT,
+       ADD COLUMN _related_entity_ids JSON,
        ADD COLUMN _status VARCHAR(32) DEFAULT 'NEW' NOT NULL,
-       ADD COLUMN _statusMsg TEXT,
+       ADD COLUMN _status_message TEXT,
        ADD COLUMN _id INT PRIMARY KEY NOT NULL AUTO_INCREMENT"
     );
   }
 
+  /**
+   * Has the import job completed.
+   *
+   * @return bool
+   *   True if no rows remain to be imported.
+   *
+   * @throws \API_Exception
+   * @throws \CRM_Core_Exception
+   */
+  public function isCompleted(): bool {
+    return (bool) $this->getRowCount(['new']);
+  }
+
+  /**
+   * Update the status of the import row to reflect the processing outcome.
+   *
+   * @param int $id
+   * @param string $status
+   * @param string $message
+   * @param int|null $entityID
+   *   Optional created entity ID
+   * @param array $relatedEntityIDs
+   *   Optional array e.g ['related_contact' => 4]
+   *
+   * @throws \API_Exception
+   * @throws \CRM_Core_Exception
+   */
+  public function updateStatus(int $id, string $status, string $message, ? int $entityID = NULL, array $relatedEntityIDs = []): void {
+    $sql = 'UPDATE ' . $this->getTableName() . ' SET _status = %1, _status_message = %2 ';
+    $params = [1 => [$status, 'String'], 2 => [$message, 'String']];
+    if ($entityID) {
+      $sql .= ', _entity_id = %3';
+      $params[3] = [$entityID, 'Integer'];
+    }
+    if ($relatedEntityIDs) {
+      $sql .= ', _related_entities = %4';
+      $params[4] = [json_encode($relatedEntityIDs), 'String'];
+    }
+    CRM_Core_DAO::executeQuery($sql . ' WHERE _id = ' . $id, $params);
+  }
+
   /**
    *
    * @throws \API_Exception
@@ -473,6 +516,7 @@ protected function getStatusMapping(): array {
       CRM_Import_Parser::DUPLICATE => ['duplicate'],
       CRM_Import_Parser::NO_MATCH => ['invalid_no_match'],
       CRM_Import_Parser::UNPARSED_ADDRESS_WARNING => ['warning_unparsed_address'],
+      'new' => ['new'],
     ];
   }
 

CRM/Import/Forms.php

@@ -518,6 +518,10 @@ public static function reorderOutput(array $record): array {
     $message = array_pop($record);
     // Also pop off the status - but we are not going to use this at this stage.
     array_pop($record);
+    // Related entities
+    array_pop($record);
+    // Entity_id
+    array_pop($record);
     array_unshift($record, $message);
     array_unshift($record, $rowNumber);
     return $record;