2028 - Sort heading for CSV Export

application/classes/Ushahidi/Formatter/Post/CSV.php

@@ -37,113 +37,223 @@ public function __invoke($records)
 	 */
 	protected function generateCSVRecords($records)
 	{
-		// Get CSV heading
-		$heading = $this->getCSVHeading($records);
-		// Sort the columns from the heading so that they match with the record keys
-		ksort($heading);
 
+		/**
+		 * Get the columns from the heading, already sorted to match the key's stage & priority.
+		 */
+		$heading = $this->getCSVHeading($records);
 		// Send response as CSV download
 		header('Access-Control-Allow-Origin: *');
 		header('Content-Type: text/csv; charset=utf-8');
 		header('Cache-Control', 'no-cache, no-store, max-age=0, must-revalidate');
 
 		$fp = fopen('php://output', 'w');
+		/**
+		 * Before doing anything, clean the ouput buffer and avoid garbage like unnecessary space paddings in our csv export
+		 */
+		ob_clean();
 
 		// Add heading
-		fputcsv($fp, $heading);
+		fputcsv($fp, array_values($heading));
 
 		foreach ($records as $record)
 		{
-			unset($record['attributes']);
-
 			// Transform post_date to a string
 			if ($record['post_date'] instanceof \DateTimeInterface) {
 				$record['post_date'] = $record['post_date']->format("Y-m-d H:i:s");
 			}
-
-			foreach ($record as $key => $val)
-			{
-				// Assign form values
-				if ($key == 'values')
-				{
-					unset($record[$key]);
-
-					foreach ($val as $key => $val)
-					{
-						$this->assignRowValue($record, $key, $val[0]);
-					}
-				}
-
-				// Assign post values
-				else
-				{
-					unset($record[$key]);
-					$this->assignRowValue($record, $key, $val);
-				}
+			$values = [];
+			foreach ($heading as $key => $value) {
+				$values[] = $this->getValueFromRecord($record, $key);
 			}
-
-			// Pad record
-			$missing_keys = array_diff(array_keys($heading), array_keys($record));
-			$record = array_merge($record, array_fill_keys($missing_keys, null));
-
-			// Sort the keys so that they match with columns from the CSV heading
-			ksort($record);
-
-			fputcsv($fp, $record);
+			fputcsv($fp, $values);
 		}
-
 		fclose($fp);
 
 		// No need for further processing
 		exit;
 	}
 
-	private function assignRowValue(&$record, $key, $value)
-	{
-		if (is_array($value))
-		{
-			// Assign in multiple columns
-			foreach ($value as $sub_key => $sub_value)
-			{
-				$record[$key.'.'.$sub_key] = $sub_value;
-			}
+	private function getValueFromRecord($record, $keyParam){
+		$return = '';
+		$keySet = explode('.', $keyParam); //contains key + index of the key, if any
+		$headingKey = $keySet[0];
+		$key = isset($keySet[1]) ? $keySet[1] : null;
+		$recordValue = isset ($record['attributes']) && isset($record['attributes'][$headingKey])? $record['values']: $record;
+		if($key === 'lat' || $key === 'lon'){
+			/*
+			 * Lat/Lon are never multivalue fields so we can get the first index  only
+			 */
+			$return = isset($recordValue[$headingKey][0][$key])? ($recordValue[$headingKey][0][$key]): '';
+		} else if ($key !== null) {
+			/**
+			 * we work with multiple posts which means our actual count($record[$key])
+			 * value might not exist in all of the posts we are posting in the CSV
+			 */
+			$return = isset($recordValue[$headingKey][$key])? ($recordValue[$headingKey][$key]): '';
+		} else{
+			$emptyRecord = !isset($record[$headingKey]) || (is_array($record[$headingKey]) && empty($record[$headingKey]));
+			$return = $emptyRecord ? '' : $record[$headingKey];
 		}
+		return $return;
+	}
 
-		// ... else assign value as single string
-		else
-		{
-			$record[$key] = $value;
-		}
+	/**
+	 * @param $fields: an array with the form: ["key": (value)] where value can be anything that the user chose.
+	 * @return array of sorted fields with a zero based index. Multivalue keys have the format keyxyz.index index being an arbitrary count of the amount of fields.
+	 */
+	private function createSortedHeading($fields){
+		$headingResult = [];
+		// fieldsWithPriorityValue: an associative array with the form ["uuid"=>[label: string, priority: number, stage: number],"uuid"=>[label: string, priority: number, stage: number]]
+		$fieldsWithPriorityValue = [];
+		/**
+		 * Assign $heading and $fieldsWithpriorityValue.
+		 */
+		$this->setPriorityAndNativeFieldArrays($headingResult, $fieldsWithPriorityValue, $fields);
+		/**
+		 * Sort the non custom priority fields alphabetically, ASC (default)
+		 */
+		uasort($headingResult, function($item1, $item2){
+			return strcmp($item1, $item2);
+		});
+		/**
+		 * sorting the multidimensional array of properties
+		 */
+		/**
+		 * First, group fields by stage and survey id
+		 */
+		$attributeKeysWithStage = $this->groupFieldsByStage($fieldsWithPriorityValue);
+		/**
+		 * After we have group by stage , we can proceed to sort each field by priority inside the stage
+		 */
+		$attributeKeysWithStageFlat = $this->sortGroupedFieldsByPriority($attributeKeysWithStage);
+		/**
+		 * Add the custom priority fields to the heading array and return it, as is.
+		 */
+		$headingResult += $attributeKeysWithStageFlat;
+		return $headingResult;
 	}
 
-	private function assignColumnHeading(&$columns, $key, $label, $value)
-	{
-		if (is_array($value))
-		{
-			// Assign in multiple columns
-			foreach ($value as $sub_key => $sub_value)
-			{
-				$multivalue_key = $key.'.'.$sub_key;
+	/**
+	 * Separate by fields that have custom priority and fields that do not have custom priority assigned
+	 * @param $headingResult by reference. => used for regular post fields (native)
+	 * @param $fieldsWithPriorityValue by reference. => used for fields that have a priority value
+	 * @param $fields
+	 */
+	private function setPriorityAndNativeFieldArrays(&$headingResult, &$fieldsWithPriorityValue, $fields){
+		foreach ($fields as $fieldKey => $fieldAttr) {
+			if (!is_array($fieldAttr)) {
+				$headingResult[$fieldKey] = $fieldAttr;
+			} else if (isset($fieldAttr['nativeField'])){
+				$headingResult = $this->addNativeFieldToHeading($headingResult, $fieldAttr, $fieldKey);
+			} else {
+				$fieldsWithPriorityValue[$fieldKey] = $fieldAttr;
+			}
+		}
+	}
 
-				if (! in_array($multivalue_key, $columns))
-				{
-					$columns[$multivalue_key] = $label.'.'.$sub_key;
+	/**
+	 * @param $heading: the CSV heading field.
+	 * @param $attr: the field itself to get the new heading item's label and total count (max usage in a single post).
+	 * @param $key the heading key. $heading will use this key or a variation for multi value keys (key.index) depending on the count property of $attr
+	 * @return $heading: the csv heading field, with a new key in it (single or multi value key)
+	 */
+	private function addNativeFieldToHeading($heading, $attr, $key) {
+		if ($attr['count'] === 0) {
+			$heading[$key] = $attr['label'];
+		}
+		for ($i = 0 ; $i < $attr['count']; $i++){
+			$heading[$key.'.'.$i] = $attr['label'].'.'.$i;
+		}
+		return $heading;
+	}
+	/**
+	 * @param $groupedFields is an associative array with fields grouped in arrays by their stage
+	 * @return array . Flat, associative. Example => ['keyxyz'=>'label for key', 'keyxyz2'=>'label for key2']
+	 */
+	private function sortGroupedFieldsByPriority($groupedFields){
+		$attributeKeysWithStageFlat = [];
+		foreach ($groupedFields as $stageKey => $attributeKeys){
+			/**
+			 * uasort is used here to preserve the associative array keys when they are sorted
+			 */
+			uasort($attributeKeys, function ($item1, $item2) {
+				if ($item1['priority'] === $item2['priority']){
+					/**
+					 * if they are the same in priority, then that maeans we will fall back to alphabetical priority for them
+					 */
+					return $item1['label'] < $item2['label'] ? -1 : 1;
+				}
+				return $item1['priority'] < $item2['priority'] ? -1 : 1;
+			});
+			/**
+			 * Finally, we can flatten the array, and set the fields (key->labels) with the user-selected order.
+			 */
+			foreach ($attributeKeys as $attributeKey => $attribute){
+				if (is_array($attribute) && isset($attribute['count']) && $attribute['type'] !== 'point'){
+					/**
+					 * If the attribute has a count key, it means we want to show that as key.index in the header.
+					 * This is to make sure we don't miss values in multi-value fields
+					 */
+					for ($i = 0 ; $i < $attribute['count']; $i++){
+						$attributeKeysWithStageFlat[$attributeKey.'.'.$i] = $attribute['label'].'.'.$i;
+					}
+				} else if (isset($attribute['type']) && $attribute['type'] === 'point'){
+					$attributeKeysWithStageFlat[$attributeKey.'.lat'] = $attribute['label'].'.lat';
+					$attributeKeysWithStageFlat[$attributeKey.'.lon'] = $attribute['label'].'.lon';
 				}
 			}
 		}
+		return $attributeKeysWithStageFlat;
+	}
+	/**
+	 * @desc Group fields by their stage in the form.
+	 * @param $fields
+	 * @return array (associative) . Example structure => ie ['stg1'=>['att1'=> obj, 'att2'=> obj],'stg2'=>['att3'=> obj, 'att4'=> obj],]
+	 *
+	 */
+	private function groupFieldsByStage($fields) {
+		$attributeKeysWithStage = [];
 
-		// ... else assign single key
-		else
-		{
-			if (! in_array($key, $columns))
-			{
-				$columns[$key] = $label;
+		foreach ($fields as $attributeKey => $attribute){
+			$key = $attribute["form_id"]."".$attribute["stage"];
+			if (!array_key_exists($key, $attributeKeysWithStage)){
+				$attributeKeysWithStage[$key] = [];
+			}
+			$attributeKeysWithStage[$key][$attributeKey] = $attribute;
+		}
+		ksort($attributeKeysWithStage);
+		return $attributeKeysWithStage;
+
+	}
+
+	/**
+	 * @param $columns by reference .
+	 * @param $key
+	 * @param $label
+	 * @param $value
+	 */
+	private function assignColumnHeading(&$columns, $key, $labelObject, $value)
+	{
+		$prevColumnValue = isset($columns[$key]) ? $columns[$key]: ['count' => 0];
+		/**
+		 * If $value is an array, then that might mean it has multiple values.
+		 * We want to count the values to make sure we use the right key format and can return all results in the CSV
+		 */
+		if (is_array($value)){
+			$headingCount = $prevColumnValue['count'] < count($value)?  count($value) : $prevColumnValue['count'] ;
+			if (!is_array($labelObject)){
+				$labelObject = ['label' => $labelObject, 'count' => $headingCount, 'type' => null, 'nativeField' => true];
 			}
+			$labelObject['count'] = $headingCount;
 		}
+		$columns[$key] = $labelObject;
 	}
 
 	/**
-	 * Extracts column names shared across posts to create a CSV heading
+	 * Extracts column names shared across posts to create a CSV heading, and sorts them with the following criteria:
+	 * - Survey "native" fields such as title from the post table go first. These are sorted alphabetically.
+	 * - Form_attributes are grouped by stage, and sorted in ASC order by priority
 	 *
 	 * @param array $records
 	 *
@@ -156,8 +266,6 @@ protected function getCSVHeading($records)
 		// Collect all column headings
 		foreach ($records as $record)
 		{
-			//$record = $record->asArray();
-
 			$attributes = $record['attributes'];
 			unset($record['attributes']);
 
@@ -169,11 +277,9 @@ protected function getCSVHeading($records)
 
 					foreach ($val as $key => $val)
 					{
-						$label = $attributes[$key];
-						$this->assignColumnHeading($columns, $key, $label, $val[0]);
+						$this->assignColumnHeading($columns, $key, $attributes[$key], $val);
 					}
 				}
-
 				// Assign post keys
 				else
 				{
@@ -182,7 +288,7 @@ protected function getCSVHeading($records)
 			}
 		}
 
-		return $columns;
+		return $this->createSortedHeading($columns);
 	}
 
 	/**

application/classes/Ushahidi/Repository/Post/Export.php

@@ -14,23 +14,28 @@
 class Ushahidi_Repository_Post_Export extends Ushahidi_Repository_Post
 {
 
+	/**
+	 * @param $data
+	 * @return array
+	 */
   public function retrieveColumnNameData($data) {
 
     // Set attribute keys
     $attributes = [];
-		foreach ($data['values'] as $key => $val)
+	foreach ($data['values'] as $key => $val)
     {
-      $attribute = $this->form_attribute_repo->getByKey($key);
-      $attributes[$key] = $attribute->label;
+        $attribute = $this->form_attribute_repo->getByKey($key);
+	 	$attributes[$key] = ['label' => $attribute->label, 'priority'=> $attribute->priority, 'stage' => $attribute->form_stage_id, 'type'=> $attribute->type, 'form_id'=> $data['form_id']];
 
-      // Set attribute names
-      if ($attribute->type === 'tags') {
-        $data['values'][$key] = $this->retrieveTagNames($val);
-      }
+		// Set attribute names
+		if ($attribute->type === 'tags') {
+			$data['values'][$key] = $this->retrieveTagNames($val);
+		}
     }
 
     $data += ['attributes' => $attributes];
 
+
     // Set Set names
     if (!empty($data['sets'])) {
       $data['sets'] = $this->retrieveSetNames($data['sets']);
@@ -46,13 +51,7 @@ public function retrieveColumnNameData($data) {
       $form = $this->form_repo->get($data['form_id']);
       $data['form_name'] = $form->name;
     }
-
-    if (!empty($data['tags'])) {
-      $data['tags'] = $this->retrieveTagNames($data['tags']);
-    }
-
     return $data;
-
   }
 
   public function retrieveTagNames($tag_ids) {

tests/datasets/ushahidi/Base.yml

@@ -436,7 +436,7 @@ form_attributes:
     form_stage_id: 7
   -
     id: 24
-    label: "Test Field Level Locking 7"
+    label: "A Test Field Level Locking 7"
     key: "test_field_locking_visible_4"
     type: "varchar"
     input: "text"