diff --git a/application/classes/Ushahidi/Formatter/Post/CSV.php b/application/classes/Ushahidi/Formatter/Post/CSV.php index fbcf155cc8..b3376d9bd0 100644 --- a/application/classes/Ushahidi/Formatter/Post/CSV.php +++ b/application/classes/Ushahidi/Formatter/Post/CSV.php @@ -37,113 +37,223 @@ public function __invoke($records) */ protected function generateCSVRecords($records) { - // Get CSV heading - $heading = $this->getCSVHeading($records); - // Sort the columns from the heading so that they match with the record keys - ksort($heading); + /** + * Get the columns from the heading, already sorted to match the key's stage & priority. + */ + $heading = $this->getCSVHeading($records); // Send response as CSV download header('Access-Control-Allow-Origin: *'); header('Content-Type: text/csv; charset=utf-8'); header('Cache-Control', 'no-cache, no-store, max-age=0, must-revalidate'); $fp = fopen('php://output', 'w'); + /** + * Before doing anything, clean the ouput buffer and avoid garbage like unnecessary space paddings in our csv export + */ + ob_clean(); // Add heading - fputcsv($fp, $heading); + fputcsv($fp, array_values($heading)); foreach ($records as $record) { - unset($record['attributes']); - // Transform post_date to a string if ($record['post_date'] instanceof \DateTimeInterface) { $record['post_date'] = $record['post_date']->format("Y-m-d H:i:s"); } - - foreach ($record as $key => $val) - { - // Assign form values - if ($key == 'values') - { - unset($record[$key]); - - foreach ($val as $key => $val) - { - $this->assignRowValue($record, $key, $val[0]); - } - } - - // Assign post values - else - { - unset($record[$key]); - $this->assignRowValue($record, $key, $val); - } + $values = []; + foreach ($heading as $key => $value) { + $values[] = $this->getValueFromRecord($record, $key); } - - // Pad record - $missing_keys = array_diff(array_keys($heading), array_keys($record)); - $record = array_merge($record, array_fill_keys($missing_keys, null)); - - // Sort the keys so that they match with columns from the CSV heading - ksort($record); - - fputcsv($fp, $record); + fputcsv($fp, $values); } - fclose($fp); // No need for further processing exit; } - private function assignRowValue(&$record, $key, $value) - { - if (is_array($value)) - { - // Assign in multiple columns - foreach ($value as $sub_key => $sub_value) - { - $record[$key.'.'.$sub_key] = $sub_value; - } + private function getValueFromRecord($record, $keyParam){ + $return = ''; + $keySet = explode('.', $keyParam); //contains key + index of the key, if any + $headingKey = $keySet[0]; + $key = isset($keySet[1]) ? $keySet[1] : null; + $recordValue = isset ($record['attributes']) && isset($record['attributes'][$headingKey])? $record['values']: $record; + if($key === 'lat' || $key === 'lon'){ + /* + * Lat/Lon are never multivalue fields so we can get the first index only + */ + $return = isset($recordValue[$headingKey][0][$key])? ($recordValue[$headingKey][0][$key]): ''; + } else if ($key !== null) { + /** + * we work with multiple posts which means our actual count($record[$key]) + * value might not exist in all of the posts we are posting in the CSV + */ + $return = isset($recordValue[$headingKey][$key])? ($recordValue[$headingKey][$key]): ''; + } else{ + $emptyRecord = !isset($record[$headingKey]) || (is_array($record[$headingKey]) && empty($record[$headingKey])); + $return = $emptyRecord ? '' : $record[$headingKey]; } + return $return; + } - // ... else assign value as single string - else - { - $record[$key] = $value; - } + /** + * @param $fields: an array with the form: ["key": (value)] where value can be anything that the user chose. + * @return array of sorted fields with a zero based index. Multivalue keys have the format keyxyz.index index being an arbitrary count of the amount of fields. + */ + private function createSortedHeading($fields){ + $headingResult = []; + // fieldsWithPriorityValue: an associative array with the form ["uuid"=>[label: string, priority: number, stage: number],"uuid"=>[label: string, priority: number, stage: number]] + $fieldsWithPriorityValue = []; + /** + * Assign $heading and $fieldsWithpriorityValue. + */ + $this->setPriorityAndNativeFieldArrays($headingResult, $fieldsWithPriorityValue, $fields); + /** + * Sort the non custom priority fields alphabetically, ASC (default) + */ + uasort($headingResult, function($item1, $item2){ + return strcmp($item1, $item2); + }); + /** + * sorting the multidimensional array of properties + */ + /** + * First, group fields by stage and survey id + */ + $attributeKeysWithStage = $this->groupFieldsByStage($fieldsWithPriorityValue); + /** + * After we have group by stage , we can proceed to sort each field by priority inside the stage + */ + $attributeKeysWithStageFlat = $this->sortGroupedFieldsByPriority($attributeKeysWithStage); + /** + * Add the custom priority fields to the heading array and return it, as is. + */ + $headingResult += $attributeKeysWithStageFlat; + return $headingResult; } - private function assignColumnHeading(&$columns, $key, $label, $value) - { - if (is_array($value)) - { - // Assign in multiple columns - foreach ($value as $sub_key => $sub_value) - { - $multivalue_key = $key.'.'.$sub_key; + /** + * Separate by fields that have custom priority and fields that do not have custom priority assigned + * @param $headingResult by reference. => used for regular post fields (native) + * @param $fieldsWithPriorityValue by reference. => used for fields that have a priority value + * @param $fields + */ + private function setPriorityAndNativeFieldArrays(&$headingResult, &$fieldsWithPriorityValue, $fields){ + foreach ($fields as $fieldKey => $fieldAttr) { + if (!is_array($fieldAttr)) { + $headingResult[$fieldKey] = $fieldAttr; + } else if (isset($fieldAttr['nativeField'])){ + $headingResult = $this->addNativeFieldToHeading($headingResult, $fieldAttr, $fieldKey); + } else { + $fieldsWithPriorityValue[$fieldKey] = $fieldAttr; + } + } + } - if (! in_array($multivalue_key, $columns)) - { - $columns[$multivalue_key] = $label.'.'.$sub_key; + /** + * @param $heading: the CSV heading field. + * @param $attr: the field itself to get the new heading item's label and total count (max usage in a single post). + * @param $key the heading key. $heading will use this key or a variation for multi value keys (key.index) depending on the count property of $attr + * @return $heading: the csv heading field, with a new key in it (single or multi value key) + */ + private function addNativeFieldToHeading($heading, $attr, $key) { + if ($attr['count'] === 0) { + $heading[$key] = $attr['label']; + } + for ($i = 0 ; $i < $attr['count']; $i++){ + $heading[$key.'.'.$i] = $attr['label'].'.'.$i; + } + return $heading; + } + /** + * @param $groupedFields is an associative array with fields grouped in arrays by their stage + * @return array . Flat, associative. Example => ['keyxyz'=>'label for key', 'keyxyz2'=>'label for key2'] + */ + private function sortGroupedFieldsByPriority($groupedFields){ + $attributeKeysWithStageFlat = []; + foreach ($groupedFields as $stageKey => $attributeKeys){ + /** + * uasort is used here to preserve the associative array keys when they are sorted + */ + uasort($attributeKeys, function ($item1, $item2) { + if ($item1['priority'] === $item2['priority']){ + /** + * if they are the same in priority, then that maeans we will fall back to alphabetical priority for them + */ + return $item1['label'] < $item2['label'] ? -1 : 1; + } + return $item1['priority'] < $item2['priority'] ? -1 : 1; + }); + /** + * Finally, we can flatten the array, and set the fields (key->labels) with the user-selected order. + */ + foreach ($attributeKeys as $attributeKey => $attribute){ + if (is_array($attribute) && isset($attribute['count']) && $attribute['type'] !== 'point'){ + /** + * If the attribute has a count key, it means we want to show that as key.index in the header. + * This is to make sure we don't miss values in multi-value fields + */ + for ($i = 0 ; $i < $attribute['count']; $i++){ + $attributeKeysWithStageFlat[$attributeKey.'.'.$i] = $attribute['label'].'.'.$i; + } + } else if (isset($attribute['type']) && $attribute['type'] === 'point'){ + $attributeKeysWithStageFlat[$attributeKey.'.lat'] = $attribute['label'].'.lat'; + $attributeKeysWithStageFlat[$attributeKey.'.lon'] = $attribute['label'].'.lon'; } } } + return $attributeKeysWithStageFlat; + } + /** + * @desc Group fields by their stage in the form. + * @param $fields + * @return array (associative) . Example structure => ie ['stg1'=>['att1'=> obj, 'att2'=> obj],'stg2'=>['att3'=> obj, 'att4'=> obj],] + * + */ + private function groupFieldsByStage($fields) { + $attributeKeysWithStage = []; - // ... else assign single key - else - { - if (! in_array($key, $columns)) - { - $columns[$key] = $label; + foreach ($fields as $attributeKey => $attribute){ + $key = $attribute["form_id"]."".$attribute["stage"]; + if (!array_key_exists($key, $attributeKeysWithStage)){ + $attributeKeysWithStage[$key] = []; + } + $attributeKeysWithStage[$key][$attributeKey] = $attribute; + } + ksort($attributeKeysWithStage); + return $attributeKeysWithStage; + + } + + /** + * @param $columns by reference . + * @param $key + * @param $label + * @param $value + */ + private function assignColumnHeading(&$columns, $key, $labelObject, $value) + { + $prevColumnValue = isset($columns[$key]) ? $columns[$key]: ['count' => 0]; + /** + * If $value is an array, then that might mean it has multiple values. + * We want to count the values to make sure we use the right key format and can return all results in the CSV + */ + if (is_array($value)){ + $headingCount = $prevColumnValue['count'] < count($value)? count($value) : $prevColumnValue['count'] ; + if (!is_array($labelObject)){ + $labelObject = ['label' => $labelObject, 'count' => $headingCount, 'type' => null, 'nativeField' => true]; } + $labelObject['count'] = $headingCount; } + $columns[$key] = $labelObject; } /** - * Extracts column names shared across posts to create a CSV heading + * Extracts column names shared across posts to create a CSV heading, and sorts them with the following criteria: + * - Survey "native" fields such as title from the post table go first. These are sorted alphabetically. + * - Form_attributes are grouped by stage, and sorted in ASC order by priority * * @param array $records * @@ -156,8 +266,6 @@ protected function getCSVHeading($records) // Collect all column headings foreach ($records as $record) { - //$record = $record->asArray(); - $attributes = $record['attributes']; unset($record['attributes']); @@ -169,11 +277,9 @@ protected function getCSVHeading($records) foreach ($val as $key => $val) { - $label = $attributes[$key]; - $this->assignColumnHeading($columns, $key, $label, $val[0]); + $this->assignColumnHeading($columns, $key, $attributes[$key], $val); } } - // Assign post keys else { @@ -182,7 +288,7 @@ protected function getCSVHeading($records) } } - return $columns; + return $this->createSortedHeading($columns); } /** diff --git a/application/classes/Ushahidi/Repository/Post/Export.php b/application/classes/Ushahidi/Repository/Post/Export.php index 5de77c8d42..f56056d98b 100644 --- a/application/classes/Ushahidi/Repository/Post/Export.php +++ b/application/classes/Ushahidi/Repository/Post/Export.php @@ -14,23 +14,28 @@ class Ushahidi_Repository_Post_Export extends Ushahidi_Repository_Post { + /** + * @param $data + * @return array + */ public function retrieveColumnNameData($data) { // Set attribute keys $attributes = []; - foreach ($data['values'] as $key => $val) + foreach ($data['values'] as $key => $val) { - $attribute = $this->form_attribute_repo->getByKey($key); - $attributes[$key] = $attribute->label; + $attribute = $this->form_attribute_repo->getByKey($key); + $attributes[$key] = ['label' => $attribute->label, 'priority'=> $attribute->priority, 'stage' => $attribute->form_stage_id, 'type'=> $attribute->type, 'form_id'=> $data['form_id']]; - // Set attribute names - if ($attribute->type === 'tags') { - $data['values'][$key] = $this->retrieveTagNames($val); - } + // Set attribute names + if ($attribute->type === 'tags') { + $data['values'][$key] = $this->retrieveTagNames($val); + } } $data += ['attributes' => $attributes]; + // Set Set names if (!empty($data['sets'])) { $data['sets'] = $this->retrieveSetNames($data['sets']); @@ -46,13 +51,7 @@ public function retrieveColumnNameData($data) { $form = $this->form_repo->get($data['form_id']); $data['form_name'] = $form->name; } - - if (!empty($data['tags'])) { - $data['tags'] = $this->retrieveTagNames($data['tags']); - } - return $data; - } public function retrieveTagNames($tag_ids) { diff --git a/tests/datasets/ushahidi/Base.yml b/tests/datasets/ushahidi/Base.yml index e2a02374f2..7278da383c 100644 --- a/tests/datasets/ushahidi/Base.yml +++ b/tests/datasets/ushahidi/Base.yml @@ -436,7 +436,7 @@ form_attributes: form_stage_id: 7 - id: 24 - label: "Test Field Level Locking 7" + label: "A Test Field Level Locking 7" key: "test_field_locking_visible_4" type: "varchar" input: "text" diff --git a/tests/integration/bootstrap/RestContext.php b/tests/integration/bootstrap/RestContext.php index 6baf156de3..93253b58ad 100644 --- a/tests/integration/bootstrap/RestContext.php +++ b/tests/integration/bootstrap/RestContext.php @@ -12,6 +12,7 @@ */ use Behat\Behat\Context\Context; +use Behat\Gherkin\Node\PyStringNode; use Symfony\Component\Yaml\Yaml; use stdClass; @@ -260,6 +261,32 @@ public function iRequest($pageUrl) } } + /** + * @Then the csv response body should have heading: + */ + public function theCsvResponseBodyShouldHaveHeading(PyStringNode $string) + { + $data = $this->response->getBody(true); + $data = explode("\n", $data); + if (!$data[0] || $data[0] !== $string->getRaw()) { + throw new \Exception("Response {{$data[0]}} \n did not match \n{{$string->getRaw()}}"); + } + } + + /** + * @Then the csv response body should have :arg1 columns in row :arg2 + */ + public function theCsvResponseBodyShouldHaveColumnsInRow($arg1, $arg2) + { + $data = $this->response->getBody(true); + $rows = explode("\n", $data); + $columnCount = count(explode(",", $rows[$arg2])); + if ($columnCount !== intval($arg1)) { + throw new \Exception("Row $arg2 should have $arg1 columns. Found $columnCount"); + } + } + + /** * @Then /^the response is JSON$/ */ diff --git a/tests/integration/export.feature b/tests/integration/export.feature new file mode 100644 index 0000000000..a47f5bf9df --- /dev/null +++ b/tests/integration/export.feature @@ -0,0 +1,13 @@ +@post @oauth2Skip +Feature: Testing the Export API + @resetFixture @csvexport + Scenario: Search All Posts and export the results + Given that I want to get all "Posts" + When I request "/posts/export" + And that the response "Content-Type" header is "text/csv" + Then the csv response body should have heading: + """ + author_email,author_realname,color,completed_stages.0,completed_stages.1,contact_id,content,created,form_id,form_name,id,locale,message_id,parent_id,post_date,published_to.0,sets.0,slug,source,status,tags.0,tags.1,title,type,updated,user_id,"Last Location (point).lat","Last Location (point).lon","Test varchar.0","Test varchar.1",Categories.0,Categories.1,"Geometry test.0","Second Point.lat","Second Point.lon",Status.0,Links.0,Links.1,"Person Status.0","Last Location.0","Test Field Level Locking 3.0","Test Field Level Locking 4.0","Test Field Level Locking 5.0","A Test Field Level Locking 7.0","Test Field Level Locking 6.0" + """ + And the csv response body should have 45 columns in row 0 + And the csv response body should have 45 columns in row 1 \ No newline at end of file