Specify line endings and file encoding so MSMARCO passage works on Wi…

…ndows (#953)
castorini · Jan 15, 2020 · f21137b · f21137b
1 parent 51879e0
commit f21137b
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 4 deletions.
diff --git a/src/main/python/msmarco/convert_collection_to_jsonl.py b/src/main/python/msmarco/convert_collection_to_jsonl.py
@@ -22,15 +22,15 @@
 def convert_collection(args):
     print('Converting collection...')
     file_index = 0
-    with open(args.collection_path) as f:
+    with open(args.collection_path, encoding='utf-8') as f:
         for i, line in enumerate(f):
             doc_id, doc_text = line.rstrip().split('\t')
 
             if i % args.max_docs_per_file == 0:
                 if i > 0:
                     output_jsonl_file.close()
                 output_path = os.path.join(args.output_folder, 'docs{:02d}.json'.format(file_index))
-                output_jsonl_file = open(output_path, 'w')
+                output_jsonl_file = open(output_path, 'w', encoding='utf-8', newline='\n')
                 file_index += 1
             output_dict = {'id': doc_id, 'contents': doc_text}
             output_jsonl_file.write(json.dumps(output_dict) + '\n')

diff --git a/src/main/python/msmarco/filter_queries.py b/src/main/python/msmarco/filter_queries.py
@@ -31,8 +31,8 @@
             query_id, _, _, _ = line.rstrip().split('\t')
             qrels.add(query_id)
 
-    with open(args.output_queries, 'w') as fout:
-        with open(args.queries) as f:
+    with open(args.output_queries, 'w', encoding='utf-8', newline='\n') as fout:
+        with open(args.queries, encoding='utf-8') as f:
             for line in f:
                 query_id, _ = line.rstrip().split('\t')
                 if query_id in qrels: