Processing XML for enwik9 data

test/asset/raw_datasets.jsonl

@@ -46,4 +46,4 @@
 {"dataset_name": "SQuAD1", "split": "dev", "NUM_LINES": 10570, "MD5": {"train": "981b29407e0affa3b1b156f72073b945", "dev": "3e85deb501d4e538b6bc56f786231552"}, "URL": {"train": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json", "dev": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json"}, "first_line": "fd5bd80f392f3a03ec908508da3a4ea3"}
 {"dataset_name": "SQuAD2", "split": "train", "NUM_LINES": 130319, "MD5": {"train": "62108c273c268d70893182d5cf8df740", "dev": "246adae8b7002f8679c027697b0b7cf8"}, "URL": {"train": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json", "dev": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json"}, "first_line": "9b719a13e9ea95ab9700c5c631885fc8"}
 {"dataset_name": "SQuAD2", "split": "dev", "NUM_LINES": 11873, "MD5": {"train": "62108c273c268d70893182d5cf8df740", "dev": "246adae8b7002f8679c027697b0b7cf8"}, "URL": {"train": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json", "dev": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json"}, "first_line": "1e011c981d41cca284070532135eb9bd"}
-{"dataset_name": "EnWik9", "split": "train", "NUM_LINES": 13147026, "MD5": "3e773f8a1577fda2e27f871ca17f31fd", "URL": "http://mattmahoney.net/dc/enwik9.zip", "first_line": "9ac868b1ea4f13083b6c923bc3134a70"}
+{"dataset_name": "EnWik9", "split": "train", "NUM_LINES": 6348957, "MD5": "3e773f8a1577fda2e27f871ca17f31fd", "URL": "http://mattmahoney.net/dc/enwik9.zip", "first_line": "6871fddaf8c442252e4d7ad81b186a46"}

torchtext/data/datasets_utils.py

@@ -10,6 +10,7 @@
  extract_archive,
  unicode_csv_reader,
 )
+from torchtext.data.functional import custom_replace
 import codecs
 try:
  import defusedxml.ElementTree as ET
@@ -19,6 +20,61 @@
 These functions and classes are meant solely for use in torchtext.datasets and not
 for public consumption yet.
 """
+_patterns = [(r'<.*>', ''),
+ (r'&amp;', '&'),
+ (r'&lt;', '<'),
+ (r'&gt;', '>'),
+ (r'<ref[^<]*<\/ref>', ''),
+ (r'<[^>]*>', ''),
+ (r'\[http:[^] ]*', '['),
+ (r'\|thumb', ''),
+ (r'\|left', ''),
+ (r'\|right', ''),
+ (r'\|\d+px', ''),
+ (r'\[\[image:[^\[\]]*\|', ''),
+ (r'\[\[category:([^|\]]*)[^]]*\]\]', '[[$1]]'),
+ (r'\[\[[a-z\-]*:[^\]]*\]\]', ''),
+ (r'\[\[[^\|\]]*\|', '[['),
+ (r'\{\{[^\}]*\}\}', ''),
+ (r'\{[^\}]*\}', ''),
+ (r'\[', ''),
+ (r'\]', ''),
+ (r'&[^;]*;', ' '),
+ (r'A', 'a'), (r'B', 'b'), (r'C', 'c'),
+ (r'D', 'd'), (r'E', 'e'), (r'F', 'f'),
+ (r'G', 'g'), (r'H', 'h'), (r'I', 'i'),
+ (r'J', 'j'), (r'K', 'k'), (r'L', 'l'),
+ (r'M', 'm'), (r'N', 'n'), (r'O', 'o'),
+ (r'P', 'p'), (r'Q', 'q'), (r'R', 'r'),
+ (r'S', 's'), (r'T', 't'), (r'U', 'u'),
+ (r'V', 'v'), (r'W', 'w'), (r'X', 'x'),
+ (r'Y', 'y'), (r'Z', 'z'),
+ (r'0', ' zero '), (r'1', ' one '), (r'2', ' two '),
+ (r'3', ' three '), (r'4', ' four '), (r'5', ' five '),
+ (r'6', ' six '), (r'7', ' seven '), (r'8', ' eight '),
+ (r'9', ' nine '),
+ (r'[^a-z\n]+', ' '),
+ (r'\n ', ''),
+ (r'\s+', ' '),
+ (r'\n\s*\n', r'\n')
+ ]
+
+
+def _create_data_from_wikipedia_xml_dumps(input_filename):
+ # Clean wikipedia xml dumps according to https://github.com/facebookresearch/fastText/blob/master/wikifil.pl
+ norm_transform = custom_replace(_patterns)
+ with open(input_filename, 'r', encoding='utf-8') as f:
+ while True:
+ line = f.readline()
+ if not line:
+ break
+ if '#redirect' in line or '#REDIRECT' in line:
+ continue
+ line = list(norm_transform([line]))[0]
+ if line != ' ' and line != '':
+ if line[0] == ' ':
+ line = line[1:]
+ yield line.strip()
 
 
 def _clean_xml_file(f_xml):

torchtext/datasets/enwik9.py

@@ -8,15 +8,15 @@
  _wrap_split_argument,
  _add_docstring_header,
  _create_dataset_directory,
- _read_text_iterator,
+ _create_data_from_wikipedia_xml_dumps,
 )
 
 URL = 'http://mattmahoney.net/dc/enwik9.zip'
 
 MD5 = '3e773f8a1577fda2e27f871ca17f31fd'
 
 NUM_LINES = {
- 'train': 13147026
+ 'train': 6348957
 }
 
 DATASET_NAME = "EnWik9"
@@ -26,9 +26,9 @@
 @_create_dataset_directory(dataset_name=DATASET_NAME)
 @_wrap_split_argument(('train',))
 def EnWik9(root, split):
+ logging.info('Creating {} data'.format(split))
  dataset_tar = download_from_url(URL, root=root, hash_value=MD5, hash_type='md5')
  extracted_files = extract_archive(dataset_tar)
  path = extracted_files[0]
- logging.info('Creating {} data'.format(split))
  return _RawTextIterableDataset(DATASET_NAME,
- NUM_LINES[split], _read_text_iterator(path))
+ NUM_LINES[split], _create_data_from_wikipedia_xml_dumps(path))