diff --git a/README.md b/README.md index 22d6301..68120ed 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ Full list of options in `config.json`: | compression | String | No | The type of compression to apply before uploading. Supported options are `none` (default) and `gzip`. For gzipped files, the file extension will automatically be changed to `.csv.gz` for all files. | | naming_convention | String | No | (Default: None) Custom naming convention of the s3 key. Replaces tokens `date`, `stream`, and `timestamp` with the appropriate values.

Supports "folders" in s3 keys e.g. `folder/folder2/{stream}/export_date={date}/{timestamp}.csv`.

Honors the `s3_key_prefix`, if set, by prepending the "filename". E.g. naming_convention = `folder1/my_file.csv` and s3_key_prefix = `prefix_` results in `folder1/prefix_my_file.csv` | | temp_dir | String | | (Default: platform-dependent) Directory of temporary CSV files with RECORD messages. | +| flatten | Boolean | No | (Default: True) Flatten json data to columns. | ### To run tests: diff --git a/target_s3_csv/__init__.py b/target_s3_csv/__init__.py index 98fd24b..0f94c35 100644 --- a/target_s3_csv/__init__.py +++ b/target_s3_csv/__init__.py @@ -38,6 +38,7 @@ def persist_messages(messages, config, s3_client): delimiter = config.get('delimiter', ',') quotechar = config.get('quotechar', '"') + flatten = config.get('flatten', True) # Use the system specific temp directory if no custom temp_dir provided temp_dir = os.path.expanduser(config.get('temp_dir', tempfile.gettempdir())) @@ -89,7 +90,7 @@ def persist_messages(messages, config, s3_client): file_is_empty = (not os.path.isfile(filename)) or os.stat(filename).st_size == 0 - flattened_record = utils.flatten_record(record_to_load) + flattened_record = utils.flatten_record(record_to_load) if flatten else record_to_load if o['stream'] not in headers and not file_is_empty: with open(filename, 'r') as csvfile: @@ -190,4 +191,5 @@ def main(): if __name__ == '__main__': - main() \ No newline at end of file + main() +