prepare datasets for new encoding kwarg.

torchtext/datasets/ag_news.py

@@ -61,6 +61,5 @@ def AG_NEWS(root: str, split: Union[Tuple[str], str]):
     cache_dp = HttpReader(cache_dp)
     cache_dp = cache_dp.end_caching(mode="wb", same_filepath_fn=True)
 
-    # TODO: read in text mode with utf-8 encoding, see: https://github.com/pytorch/pytorch/issues/72713
-    data_dp = FileOpener(cache_dp, mode="b")
+    data_dp = FileOpener(cache_dp, encoding="utf-8")
     return data_dp.parse_csv().map(fn=lambda t: (int(t[0]), " ".join(t[1:])))

torchtext/datasets/amazonreviewfull.py

@@ -74,5 +74,5 @@ def AmazonReviewFull(root: str, split: Union[Tuple[str], str]):
     )
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
 
-    data_dp = FileOpener(cache_decompressed_dp, mode="b")
+    data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8")
     return data_dp.parse_csv().map(fn=lambda t: (int(t[0]), " ".join(t[1:])))

torchtext/datasets/amazonreviewpolarity.py

@@ -71,5 +71,5 @@ def AmazonReviewPolarity(root: str, split: Union[Tuple[str], str]):
     )
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
 
-    data_dp = FileOpener(cache_decompressed_dp, mode="b")
+    data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8")
     return data_dp.parse_csv().map(fn=lambda t: (int(t[0]), " ".join(t[1:])))

torchtext/datasets/cc100.py

@@ -164,6 +164,5 @@ def CC100(root: str, language_code: str = "en"):
     cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b").read_from_xz()
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb")
 
-    # TODO: read in text mode with utf-8 encoding, see: https://github.com/pytorch/pytorch/issues/72713
-    data_dp = FileOpener(cache_decompressed_dp, mode="b").readlines(return_path=False, decode=True)
+    data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8").readlines(return_path=False)
     return data_dp.map(lambda x: (language_code, x))

torchtext/datasets/conll2000chunking.py

@@ -72,5 +72,5 @@ def CoNLL2000Chunking(root: str, split: Union[Tuple[str], str]):
     cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b").extract(file_type="gzip")
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
 
-    data_dp = FileOpener(cache_decompressed_dp, mode="b")
-    return data_dp.readlines(decode=True).read_iob(sep=" ")
+    data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8")
+    return data_dp.readlines().read_iob(sep=" ")

torchtext/datasets/dbpedia.py

@@ -70,5 +70,5 @@ def DBpedia(root: str, split: Union[Tuple[str], str]):
     )
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
 
-    data_dp = FileOpener(cache_decompressed_dp, mode="b")
+    data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8")
     return data_dp.parse_csv().map(fn=lambda t: (int(t[0]), " ".join(t[1:])))

torchtext/datasets/enwik9.py

@@ -51,5 +51,5 @@ def EnWik9(root: str):
     cache_decompressed_dp = FileOpener(cache_decompressed_dp, mode="b").read_from_zip()
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
 
-    data_dp = FileOpener(cache_decompressed_dp, mode="b")
-    return data_dp.readlines(decode=True, return_path=False)
+    data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8")
+    return data_dp.readlines(return_path=False)

torchtext/datasets/imdb.py

@@ -80,7 +80,6 @@ def filter_imdb_data(key, fname):
         mode="wb", filepath_fn=lambda x: os.path.join(root, decompressed_folder, split, x), skip_read=True
     )
 
-    # TODO: read in text mode with utf-8 encoding, see: https://github.com/pytorch/pytorch/issues/72713
-    data_dp = FileOpener(cache_decompressed_dp, mode="b")
+    data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8")
     # get label from cache file, eg. "aclImdb_v1/train/neg" -> "neg"
-    return data_dp.readlines(decode=True).map(lambda t: (Path(t[0]).parts[-1], t[1]))
+    return data_dp.readlines().map(lambda t: (Path(t[0]).parts[-1], t[1]))

torchtext/datasets/iwslt2016.py

@@ -290,11 +290,10 @@ def IWSLT2016(
         cache_decompressed_dp, full_tgt_filepath, uncleaned_tgt_filename
     )
 
-    # TODO: read in text mode with utf-8 encoding, see: https://github.com/pytorch/pytorch/issues/72713
-    tgt_data_dp = FileOpener(cache_inner_tgt_decompressed_dp, mode="b")
-    src_data_dp = FileOpener(cache_inner_src_decompressed_dp, mode="b")
+    tgt_data_dp = FileOpener(cache_inner_tgt_decompressed_dp, encoding="utf-8")
+    src_data_dp = FileOpener(cache_inner_src_decompressed_dp, encoding="utf-8")
 
-    src_lines = src_data_dp.readlines(return_path=False, strip_newline=False, decode=True)
-    tgt_lines = tgt_data_dp.readlines(return_path=False, strip_newline=False, decode=True)
+    src_lines = src_data_dp.readlines(return_path=False, strip_newline=False)
+    tgt_lines = tgt_data_dp.readlines(return_path=False, strip_newline=False)
 
     return src_lines.zip(tgt_lines)

torchtext/datasets/iwslt2017.py

@@ -243,11 +243,10 @@ def IWSLT2017(root=".data", split=("train", "valid", "test"), language_pair=("de
         cache_decompressed_dp, full_tgt_filepath, uncleaned_tgt_filename
     )
 
-    # TODO: read in text mode with utf-8 encoding, see: https://github.com/pytorch/pytorch/issues/72713
-    tgt_data_dp = FileOpener(cache_inner_tgt_decompressed_dp, mode="b")
-    src_data_dp = FileOpener(cache_inner_src_decompressed_dp, mode="b")
+    tgt_data_dp = FileOpener(cache_inner_tgt_decompressed_dp, encoding="utf-8")
+    src_data_dp = FileOpener(cache_inner_src_decompressed_dp, encoding="utf-8")
 
-    src_lines = src_data_dp.readlines(return_path=False, strip_newline=False, decode=True)
-    tgt_lines = tgt_data_dp.readlines(return_path=False, strip_newline=False, decode=True)
+    src_lines = src_data_dp.readlines(return_path=False, strip_newline=False)
+    tgt_lines = tgt_data_dp.readlines(return_path=False, strip_newline=False)
 
     return src_lines.zip(tgt_lines)

torchtext/datasets/multi30k.py

@@ -99,11 +99,11 @@ def Multi30k(root: str, split: Union[Tuple[str], str], language_pair: Tuple[str]
     )
     tgt_cache_decompressed_dp = tgt_cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
 
-    src_data_dp = FileOpener(src_cache_decompressed_dp, mode="b").readlines(
-        decode=True, return_path=False, strip_newline=True
+    src_data_dp = FileOpener(src_cache_decompressed_dp, encoding="utf-8").readlines(
+        return_path=False, strip_newline=True
     )
-    tgt_data_dp = FileOpener(tgt_cache_decompressed_dp, mode="b").readlines(
-        decode=True, return_path=False, strip_newline=True
+    tgt_data_dp = FileOpener(tgt_cache_decompressed_dp, encoding="utf-8").readlines(
+        return_path=False, strip_newline=True
     )
 
     return src_data_dp.zip(tgt_data_dp)

torchtext/datasets/penntreebank.py

@@ -64,7 +64,6 @@ def PennTreebank(root, split: Union[Tuple[str], str]):
     )
     cache_dp = HttpReader(cache_dp).end_caching(mode="wb", same_filepath_fn=True)
 
-    # TODO: read in text mode with utf-8 encoding, see: https://github.com/pytorch/pytorch/issues/72713
-    data_dp = FileOpener(cache_dp, mode="b")
+    data_dp = FileOpener(cache_dp, encoding="utf-8")
     # remove single leading and trailing space from the dataset
-    return data_dp.readlines(return_path=False, decode=True).map(lambda t: t.strip())
+    return data_dp.readlines(return_path=False).map(lambda t: t.strip())

torchtext/datasets/sogounews.py

@@ -74,5 +74,5 @@ def SogouNews(root: str, split: Union[Tuple[str], str]):
     )
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
 
-    data_dp = FileOpener(cache_decompressed_dp, mode="b")
+    data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8")
     return data_dp.parse_csv().map(fn=lambda t: (int(t[0]), " ".join(t[1:])))

torchtext/datasets/squad1.py

@@ -61,5 +61,5 @@ def SQuAD1(root: str, split: Union[Tuple[str], str]):
         hash_type="md5",
     )
     cache_dp = HttpReader(cache_dp).end_caching(mode="wb", same_filepath_fn=True)
-    cache_dp = FileOpener(cache_dp, mode="b")
+    cache_dp = FileOpener(cache_dp, encoding="utf-8")
     return cache_dp.parse_json_files().read_squad()

torchtext/datasets/squad2.py

@@ -62,5 +62,5 @@ def SQuAD2(root: str, split: Union[Tuple[str], str]):
         hash_type="md5",
     )
     cache_dp = HttpReader(cache_dp).end_caching(mode="wb", same_filepath_fn=True)
-    cache_dp = FileOpener(cache_dp, mode="b")
+    cache_dp = FileOpener(cache_dp, encoding="utf-8")
     return cache_dp.parse_json_files().read_squad()

torchtext/datasets/sst2.py

@@ -77,7 +77,7 @@ def SST2(root, split):
     )
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
 
-    data_dp = FileOpener(cache_decompressed_dp, mode="b")
+    data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8")
     # test split for SST2 doesn't have labels
     if split == "test":
         parsed_data = data_dp.parse_csv(skip_lines=1, delimiter="\t").map(lambda t: (t[1].strip(),))

torchtext/datasets/udpos.py

@@ -65,5 +65,5 @@ def UDPOS(root: str, split: Union[Tuple[str], str]):
     )
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
 
-    data_dp = FileOpener(cache_decompressed_dp, mode="b")
-    return data_dp.readlines(decode=True).read_iob()
+    data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8")
+    return data_dp.readlines().read_iob()

torchtext/datasets/wikitext103.py

@@ -69,5 +69,5 @@ def WikiText103(root: str, split: Union[Tuple[str], str]):
         FileOpener(cache_decompressed_dp, mode="b").read_from_zip().filter(lambda x: _EXTRACTED_FILES[split] in x[0])
     )
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
-    data_dp = FileOpener(cache_decompressed_dp, mode="b")
-    return data_dp.readlines(strip_newline=False, decode=True, return_path=False)
+    data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8")
+    return data_dp.readlines(strip_newline=False, return_path=False)

torchtext/datasets/wikitext2.py

@@ -69,5 +69,5 @@ def WikiText2(root: str, split: Union[Tuple[str], str]):
         FileOpener(cache_decompressed_dp, mode="b").read_from_zip().filter(lambda x: _EXTRACTED_FILES[split] in x[0])
     )
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
-    data_dp = FileOpener(cache_decompressed_dp, mode="b")
-    return data_dp.readlines(strip_newline=False, decode=True, return_path=False)
+    data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8")
+    return data_dp.readlines(strip_newline=False, return_path=False)

torchtext/datasets/yahooanswers.py

@@ -71,6 +71,6 @@ def YahooAnswers(root: str, split: Union[Tuple[str], str]):
     cache_decompressed_dp = cache_decompressed_dp.filter(lambda x: _EXTRACTED_FILES[split] in x[0])
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
 
-    data_dp = FileOpener(cache_decompressed_dp, mode="b")
+    data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8")
 
-    return data_dp.parse_csv().map(fn=lambda t: (int(t[0]), " ".join(t[1:])))
+    return data_dp.parse_csv().map(lambda t: (int(t[0]), " ".join(t[1:])))

torchtext/datasets/yelpreviewfull.py

@@ -70,5 +70,5 @@ def YelpReviewFull(root: str, split: Union[Tuple[str], str]):
     cache_decompressed_dp = cache_decompressed_dp.read_from_tar().filter(lambda x: _EXTRACTED_FILES[split] in x[0])
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
 
-    data_dp = FileOpener(cache_decompressed_dp, mode="b")
-    return data_dp.parse_csv().map(fn=lambda t: (int(t[0]), " ".join(t[1:])))
+    data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8")
+    return data_dp.parse_csv().map(lambda t: (int(t[0]), " ".join(t[1:])))

torchtext/datasets/yelpreviewpolarity.py

@@ -71,6 +71,5 @@ def YelpReviewPolarity(root: str, split: Union[Tuple[str], str]):
 
     cache_decompressed_dp = cache_decompressed_dp.filter(lambda x: _EXTRACTED_FILES[split] in x[0])
     cache_decompressed_dp = cache_decompressed_dp.end_caching(mode="wb", same_filepath_fn=True)
-    data_dp = FileOpener(cache_decompressed_dp, mode="b")
-
-    return data_dp.parse_csv().map(fn=lambda t: (int(t[0]), " ".join(t[1:])))
+    data_dp = FileOpener(cache_decompressed_dp, encoding="utf-8")
+    return data_dp.parse_csv().map(lambda t: (int(t[0]), " ".join(t[1:])))