Format date fields to match the data dictionary on csv export (#3869)

.circleci/config.yml

@@ -7,6 +7,7 @@ commands:
           name: Install DDev
           command: |
             set -e
+            sudo rm -rf /etc/apt/sources.list.d/heroku.list
             sudo apt-get update
             sudo apt-get install ca-certificates
             curl https://apt.fury.io/drud/gpg.key | sudo apt-key add -

modules/common/src/Storage/Query.php

@@ -16,6 +16,14 @@ class Query implements
     OffsetterInterface,
     LimiterInterface {
 
+  /**
+   * The collection of records (usually, a database table) to query against.
+   *
+   * @var array
+   */
+  public $dataDictionaryFields;
+
+
   /**
    * The collection of records (usually, a database table) to query against.
    *

modules/common/src/Storage/SelectFactory.php

@@ -58,15 +58,19 @@ public function __construct(Connection $connection, string $alias = 't') {
    *   DKAN Query object.
    */
   public function create(Query $query): Select {
-    $this->dbQuery = $this->connection->select($query->collection, $this->alias);
 
+    $this->dbQuery = $this->connection->select($query->collection, $this->alias);
     $this->setQueryProperties($query);
     $this->setQueryConditions($query);
     $this->setQueryGroupBy($query);
     $this->setQueryOrderBy($query);
     $this->setQueryLimitAndOffset($query);
     $this->setQueryJoins($query);
-
+    if (!empty($query->dataDictionaryFields)) {
+      $meta_data = $query->dataDictionaryFields;
+      $fields = $this->dbQuery->getFields();
+      $this->addDateExpressions($this->dbQuery, $fields, $meta_data);
+    }
     // $string = $this->dbQuery->__toString();
     if ($query->count) {
       $this->dbQuery = $this->dbQuery->countQuery();
@@ -84,21 +88,36 @@ private function setQueryProperties(Query $query) {
     // If properties is empty, just get all from base collection.
     if (empty($query->properties)) {
       $this->dbQuery->fields($this->alias);
+
       return;
     }
-
     foreach ($query->properties as $p) {
       $this->setQueryProperty($p);
     }
   }
 
+  /**
+   * Reformatting date fields.
+   *
+   *  {@inheritdoc}
+   */
+  private function addDateExpressions($db_query, $fields, $meta_data) {
+    foreach ($meta_data as $definition) {
+      // Confirm definition name is in the fields list.
+      if ($fields[$definition['name']]['field'] ?? FALSE && $definition['type'] == 'date') {
+        $db_query->addExpression("DATE_FORMAT(" . $definition['name'] . ", '" . $definition['format'] . "')", $definition['name']);
+      }
+    }
+  }
+
   /**
    * Set a single property.
    *
    * @param mixed $property
    *   One property from a query properties array.
    */
   private function setQueryProperty($property) {
+
     if (isset($property->expression)) {
       $expressionStr = $this->expressionToString($property->expression);
       $this->dbQuery->addExpression($expressionStr, $property->alias);

modules/common/tests/src/Unit/Storage/SelectFactoryTest.php

@@ -47,6 +47,7 @@ public function testQuery(Query $query, string $sql, string $message, array $val
    */
   public function testConditionByIsEqualTo() {
     $query = new Query();
+    $query->properties = ["field1", "field2"];
     $query->conditionByIsEqualTo('prop1', 'value1');
     $db_query = $this->selectFactory->create($query);
     $this->assertStringContainsString('t.prop1 LIKE :db_condition_placeholder_0', $this->selectToString($db_query));
@@ -59,6 +60,23 @@ public function testConditionByIsEqualToCaseInsensitive() {
     $this->assertStringContainsString('t.prop1 LIKE BINARY :db_condition_placeholder_0', $this->selectToString($db_query));
   }
 
+  /**
+   * Test two variations of Query::testConditionByIsEqualTo()
+   */
+  public function testAddDateExpressions() {
+    $query = new Query();
+    $query->dataDictionaryFields = [
+      [
+        'name' => 'date',
+        'type' => 'date',
+        'format'=>'%m/%d/%Y',
+      ]
+    ];
+    $query->properties = ["date", "field2"];
+    $db_query = $this->selectFactory->create($query);
+    $this->assertStringContainsString("DATE_FORMAT(date, '%m/%d/%Y') AS date", $this->selectToString($db_query));
+  }
+
   /**
    *
    */

modules/datastore/datastore.services.yml

@@ -7,6 +7,7 @@ services:
       - '@queue'
       - '@dkan.common.job_store'
       - '@dkan.datastore.import_info_list'
+      - '@dkan.datastore.service.resource_processor.dictionary_enforcer'
 
   dkan.datastore.query:
     class: \Drupal\datastore\Service\Query

modules/datastore/src/Controller/QueryDownloadController.php

@@ -76,7 +76,7 @@ protected function streamCsvResponse(DatastoreQuery $datastoreQuery, RootedJsonD
         $this->sendRow($handle, $this->getHeaderRow($result));
 
         // Get the result pointer and send each row to the stream one by one.
-        $result = $this->queryService->runResultsQuery($datastoreQuery, FALSE);
+        $result = $this->queryService->runResultsQuery($datastoreQuery, FALSE, TRUE);
         while ($row = $result->fetchAssoc()) {
           $this->sendRow($handle, array_values($row));
         }

modules/datastore/src/Service.php

@@ -13,6 +13,7 @@
 use Drupal\datastore\Service\Factory\ImportFactoryInterface;
 use Drupal\datastore\Service\Info\ImportInfoList;
 use FileFetcher\FileFetcher;
+use Drupal\datastore\Service\ResourceProcessor\DictionaryEnforcer;
 
 /**
  * Main services for the datastore.
@@ -47,6 +48,13 @@ class Service implements ContainerInjectionInterface {
    */
   private $jobStoreFactory;
 
+  /**
+   * Datastore Query object for conversion.
+   *
+   * @var Drupal\datastore\Service\ResourceProcessor\DictionaryEnforcer
+   */
+  private $dictionaryEnforcer;
+
   /**
    * {@inheritdoc}
    */
@@ -57,6 +65,7 @@ public static function create(ContainerInterface $container) {
       $container->get('queue'),
       $container->get('dkan.common.job_store'),
       $container->get('dkan.datastore.import_info_list'),
+      $container->get('dkan.datastore.service.resource_processor.dictionary_enforcer')
     );
   }
 
@@ -73,19 +82,23 @@ public static function create(ContainerInterface $container) {
    *   Jobstore factory service.
    * @param \Drupal\datastore\Service\Info\ImportInfoList $importInfoList
    *   Import info list service.
+   * @param Drupal\datastore\Service\ResourceProcessor\DictionaryEnforcer $dictionaryEnforcer
+   *   Dictionary Enforcer object.
    */
   public function __construct(
     ResourceLocalizer $resourceLocalizer,
     ImportFactoryInterface $importServiceFactory,
     QueueFactory $queue,
     JobStoreFactory $jobStoreFactory,
-    ImportInfoList $importInfoList
+    ImportInfoList $importInfoList,
+    DictionaryEnforcer $dictionaryEnforcer
   ) {
     $this->queue = $queue;
     $this->resourceLocalizer = $resourceLocalizer;
     $this->importServiceFactory = $importServiceFactory;
     $this->jobStoreFactory = $jobStoreFactory;
     $this->importInfoList = $importInfoList;
+    $this->dictionaryEnforcer = $dictionaryEnforcer;
   }
 
   /**
@@ -180,6 +193,13 @@ public function getImportService(DataResource $resource) {
     return $this->importServiceFactory->getInstance($resource->getUniqueIdentifier(), ['resource' => $resource]);
   }
 
+  /**
+   * Returns the Data Dictionary fields.
+   */
+  public function getDataDictionaryFields() {
+    return $this->dictionaryEnforcer->returnDataDictionaryFields();
+  }
+
   /**
    * Drop a resources datastore.
    *

modules/datastore/src/Service/Query.php

@@ -26,7 +26,7 @@ class Query implements ContainerInjectionInterface {
    */
   public static function create(ContainerInterface $container) {
     return new static(
-      $container->get('dkan.datastore.service')
+      $container->get('dkan.datastore.service'),
     );
   }
 
@@ -119,16 +119,17 @@ public function getQueryStorageMap(DatastoreQuery $datastoreQuery) {
    *   DatastoreQuery object.
    * @param bool $fetch
    *   Perform fetchAll and return array if true, else just statement (cursor).
+   * @param bool $csv
+   *   Flag for csv downloads.
    *
    * @return array|\Drupal\Core\Database\StatementInterface
    *   Array of result objects or result statement of $fetch is false.
    */
-  public function runResultsQuery(DatastoreQuery $datastoreQuery, $fetch = TRUE) {
+  public function runResultsQuery(DatastoreQuery $datastoreQuery, $fetch = TRUE, $csv = FALSE) {
     $primaryAlias = $datastoreQuery->{"$.resources[0].alias"};
     if (!$primaryAlias) {
       return [];
     }
-
     $storageMap = $this->getQueryStorageMap($datastoreQuery);
 
     $storage = $storageMap[$primaryAlias];
@@ -137,8 +138,11 @@ public function runResultsQuery(DatastoreQuery $datastoreQuery, $fetch = TRUE) {
       $schema = $this->filterSchemaFields($storage->getSchema(), $storage->primaryKey());
       $datastoreQuery->{"$.properties"} = array_keys($schema['fields']);
     }
-
     $query = QueryFactory::create($datastoreQuery, $storageMap);
+    // Get data dictionary fields.
+    $meta_data = $csv != FALSE ? $this->getDatastoreService()->getDataDictionaryFields() : NULL;
+    // Pass the data dictionary metadata to the query.
+    $query->dataDictionaryFields = $csv && $meta_data ? $meta_data : NULL;
 
     $result = $storageMap[$primaryAlias]->query($query, $primaryAlias, $fetch);
 
@@ -149,6 +153,16 @@ public function runResultsQuery(DatastoreQuery $datastoreQuery, $fetch = TRUE) {
 
   }
 
+  /**
+   * Return the datastore service.
+   *
+   * @return \Drupal\datastore\Service
+   *   Datastore Service.
+   */
+  protected function getDatastoreService() {
+    return $this->datastore;
+  }
+
   /**
    * Remove the primary key from the schema field list.
    *

modules/datastore/src/Service/ResourceProcessor/DictionaryEnforcer.php

@@ -119,4 +119,20 @@ public function applyDictionary(RootedJsonData $dictionary, string $datastore_ta
       ->execute();
   }
 
+  /**
+   * Returning data dictionary fields from schema.
+   *
+   *  {@inheritdoc}
+   */
+  public function returnDataDictionaryFields() {
+    // Get DD is mode.
+    $dd_mode = $this->dataDictionaryDiscovery->getDataDictionaryMode();
+    // Get data dictionary info.
+    if ($dd_mode == "sitewide") {
+      $dict_id = $this->dataDictionaryDiscovery->getSitewideDictionaryId();
+      $metaData = $this->metastore->get('data-dictionary', $dict_id)->{"$.data.fields"};
+      return $metaData;
+    }
+  }
+
 }

modules/datastore/src/Storage/QueryFactory.php

@@ -45,7 +45,7 @@ public function __construct(DatastoreQuery $datastoreQuery, array $storageMap) {
    * @param array $storageMap
    *   Storage map array.
    *
-   * @return Drupal\common\Storage\Query
+   * @return \Drupal\common\Storage\Query
    *   DKAN query object.
    */
   public static function create(DatastoreQuery $datastoreQuery, array $storageMap): Query {

modules/datastore/tests/src/Unit/Controller/QueryDownloadControllerTest.php

@@ -24,6 +24,9 @@
 use Drupal\metastore\Storage\DataFactory;
 use Symfony\Component\DependencyInjection\ContainerInterface;
 use Symfony\Component\HttpFoundation\Request;
+use Drupal\common\Storage\SelectFactory;
+use Drupal\Core\Database\Query\Select;
+use Drupal\Tests\common\Unit\Connection;
 
 /**
  *
@@ -43,6 +46,8 @@ protected function setUp(): void {
       ->add(ContainerInterface::class, 'get', $options)
       ->add(CacheContextsManager::class, 'assertValidTokens', TRUE);
     \Drupal::setContainer($chain->getMock());
+    $this->selectFactory = $this->getSelectFactory();
+
   }
 
   protected function tearDown(): void {
@@ -55,14 +60,19 @@ protected function tearDown(): void {
    */
   private function queryResultCompare($data, $resource = NULL) {
     $request = $this->mockRequest($data);
-
+    $dataDictionaryFields = [
+      'name' => 'date',
+      'type' => 'date',
+      'format '=>'%m/%d/%Y'
+    ];
     $qController = QueryController::create($this->getQueryContainer(500));
     $response = $resource ? $qController->queryResource($resource, $request) : $qController->query($request);
     $csv = $response->getContent();
 
     $dController = QueryDownloadController::create($this->getQueryContainer(25));
     ob_start(['self', 'getBuffer']);
     $streamResponse = $resource ? $dController->queryResource($resource, $request) : $dController->query($request);
+    $streamResponse->dataDictionaryFields = $dataDictionaryFields;
     $streamResponse->sendContent();
     $streamedCsv = $this->buffer;
     ob_get_clean();
@@ -88,6 +98,48 @@ public function testStreamedQueryCsv() {
     $this->queryResultCompare($data);
   }
 
+  public function queryResultReformatted($data){
+    $request = $this->mockRequest($data);
+    $dataDictionaryFields = [
+      'name' => 'date',
+      'type' => 'date',
+      'format '=>'%m/%d/%Y'
+    ];
+    $qController = QueryController::create($this->getQueryContainer(500));
+    $response = $qController->query($request);
+    $csv = $response->getContent();
+
+    $dController = QueryDownloadController::create($this->getQueryContainer(25));
+    ob_start(['self', 'getBuffer']);
+    $streamResponse = $dController->query($request);
+    $streamResponse->dataDictionaryFields = $dataDictionaryFields;
+    //$streamResponse->sendContent();
+    $this->selectFactory->create($streamResponse);
+
+    $this->assertEquals(count(explode("\n", $csv)), count(explode("\n", $streamedCsv)));
+    $this->assertEquals($csv, $streamedCsv);
+  }
+
+  /**
+   *
+   */
+  private function getSelectFactory() {
+    return new SelectFactory($this->getConnection());
+  }
+
+  /**
+   *
+   */
+  private function getConnection() {
+    return (new Chain($this))
+      ->add(
+        Connection::class,
+        "select",
+        new Select(new Connection(new \PDO('sqlite::memory:'), []), "table", "t")
+      )
+      ->getMock();
+  }
+
   /**
    * Test streaming of a CSV file from database.
    */
@@ -382,6 +434,8 @@ private function getQueryContainer(int $rowLimit) {
       ->add(Data::class, 'getCacheMaxAge', 0)
       ->add(ConfigFactoryInterface::class, 'get', ImmutableConfig::class)
       ->add(Query::class, "getQueryStorageMap", $storageMap)
+      ->add(Query::class, 'getDatastoreService',  Service::class)
+      ->add(Service::class, 'getDataDictionaryFields', NULL)
       ->add(ImmutableConfig::class, 'get', $rowLimit);
 
     return $chain->getMock();