diff --git a/twarc/command.py b/twarc/command.py index 30e566a7..187a234c 100644 --- a/twarc/command.py +++ b/twarc/command.py @@ -213,10 +213,10 @@ def main(): # optionally create a csv writer csv_writer = None - if args.format == "csv" and command not in ["filter", "hydrate", "replies", + if args.format in ("csv", "csv-excel") and command not in ["filter", "hydrate", "replies", "retweets", "sample", "search", "timeline", "tweet"]: parser.error("csv output not available for %s" % command) - elif args.format == "csv": + elif args.format in ("csv", "csv-excel"): csv_writer = csv.writer(fh) csv_writer.writerow(get_headings()) @@ -247,6 +247,8 @@ def main(): print(json.dumps(thing), file=fh) elif (args.format == "csv"): csv_writer.writerow(get_row(thing)) + elif (args.format == "csv-excel"): + csv_writer.writerow(get_row(thing, excel=True)) logging.info("archived %s", thing['id_str']) elif 'woeid' in thing: # places @@ -321,7 +323,7 @@ def get_argparser(): parser.add_argument("--output", action="store", default=None, dest="output", help="write output to file path") parser.add_argument("--format", action="store", default="json", - dest="format", choices=["json", "csv"], + dest="format", choices=["json", "csv", "csv-excel"], help="set output format") parser.add_argument("--split", action="store", type=int, default=0, help="used with --output to split into numbered files") diff --git a/twarc/json2csv.py b/twarc/json2csv.py index 2b8bb35d..9f912741 100755 --- a/twarc/json2csv.py +++ b/twarc/json2csv.py @@ -55,7 +55,7 @@ def get_headings(): ] -def get_row(t): +def get_row(t, excel=False): get = t.get user = t.get('user').get return [ @@ -64,7 +64,7 @@ def get_row(t): get('created_at'), date_parse(get('created_at')), user('screen_name'), - text(t), + text(t) if not excel else clean_str(text(t)), tweet_type(t), coordinates(t), hashtags(t), @@ -85,13 +85,13 @@ def get_row(t): user('id_str'), user('created_at'), user('default_profile_image'), - user('description'), + user('description') if not excel else clean_str(user('description')), user('favourites_count'), user('followers_count'), user('friends_count'), user('listed_count'), - user('location'), - user('name'), + user('location') if not excel else clean_str(user('location')), + user('name') if not excel else clean_str(user('name')), user('statuses_count'), user('time_zone'), user_urls(t), @@ -99,8 +99,14 @@ def get_row(t): ] +def clean_str(string): + if isinstance(string, str): + return string.replace('\n', ' ').replace('\r', '') + return None + + def text(t): - return (t.get('full_text') or t.get('extended_tweet', {}).get('full_text') or t['text']).replace('\n', ' ') + return t.get('full_text') or t.get('extended_tweet', {}).get('full_text') or t['text'] def coordinates(t): diff --git a/utils/json2csv.py b/utils/json2csv.py index 767ac651..434764f7 100755 --- a/utils/json2csv.py +++ b/utils/json2csv.py @@ -31,6 +31,7 @@ def main(): parser.add_argument('--extra-field', '-e', help='extra fields to include. Provide a field name and a pointer to ' 'the field. Example: -e verified user.verified', nargs=2, action='append') + parser.add_argument('--excel', '-x', help='create file compatible with Excel', action='store_true') parser.add_argument('files', metavar='FILE', nargs='*', help='files to read, if empty, stdin is used') args = parser.parse_args() @@ -64,7 +65,7 @@ def main(): sheet.writerow(get_headings(extra_headings=extra_headings)) file_count += 1 tweet = json.loads(line) - sheet.writerow(get_row(tweet, extra_fields=extra_fields)) + sheet.writerow(get_row(tweet, extra_fields=extra_fields, excel=args.excel)) def numbered_filepath(filepath, num): @@ -79,8 +80,8 @@ def get_headings(extra_headings=None): return fields -def get_row(t, extra_fields=None): - row = json2csv.get_row(t) +def get_row(t, extra_fields=None, excel=False): + row = json2csv.get_row(t, excel=excel) if extra_fields: for field in extra_fields: row.append(extra_field(t, field))