Remove WMTNewsCrawl dataset back to the experimental folder because the URL link is very slow

docs/source/datasets.rst

@@ -98,12 +98,6 @@ PennTreebank
 .. autofunction:: PennTreebank
 
 
-WMTNewsCrawl
-~~~~~~~~~~~~
-
-.. autofunction:: WMTNewsCrawl
-
-
 Machine Translation
 ^^^^^^^^^^^^^^^^^^^
 

test/asset/raw_datasets.json

@@ -42,7 +42,6 @@
 {"dataset_name": "PennTreebank", "split": "train", "NUM_LINES": 42068, "MD5": {"train": "f26c4b92c5fdc7b3f8c7cdcb991d8420", "valid": "aa0affc06ff7c36e977d7cd49e3839bf", "test": "8b80168b89c18661a38ef683c0dc3721"}, "URL": {"train": "https://raw.githubusercontent.com/wojzaremba/lstm/master/data/ptb.train.txt", "test": "https://raw.githubusercontent.com/wojzaremba/lstm/master/data/ptb.test.txt", "valid": "https://raw.githubusercontent.com/wojzaremba/lstm/master/data/ptb.valid.txt"}, "first_line": " aer banknote berlitz calloway centrust cluett fromstein gitano guterman hydro-quebec ipo kia memotec mlx nahb punts rake regatta rubens sim snack-food ssangyong swapo wachter \n"}
 {"dataset_name": "PennTreebank", "split": "valid", "NUM_LINES": 3370, "MD5": {"train": "f26c4b92c5fdc7b3f8c7cdcb991d8420", "valid": "aa0affc06ff7c36e977d7cd49e3839bf", "test": "8b80168b89c18661a38ef683c0dc3721"}, "URL": {"train": "https://raw.githubusercontent.com/wojzaremba/lstm/master/data/ptb.train.txt", "test": "https://raw.githubusercontent.com/wojzaremba/lstm/master/data/ptb.test.txt", "valid": "https://raw.githubusercontent.com/wojzaremba/lstm/master/data/ptb.valid.txt"}, "first_line": " consumers may want to move their telephones a little closer to the tv set \n"}
 {"dataset_name": "PennTreebank", "split": "test", "NUM_LINES": 3761, "MD5": {"train": "f26c4b92c5fdc7b3f8c7cdcb991d8420", "valid": "aa0affc06ff7c36e977d7cd49e3839bf", "test": "8b80168b89c18661a38ef683c0dc3721"}, "URL": {"train": "https://raw.githubusercontent.com/wojzaremba/lstm/master/data/ptb.train.txt", "test": "https://raw.githubusercontent.com/wojzaremba/lstm/master/data/ptb.test.txt", "valid": "https://raw.githubusercontent.com/wojzaremba/lstm/master/data/ptb.valid.txt"}, "first_line": " no it was n't black monday \n"}
-{"dataset_name": "WMTNewsCrawl", "split": "train", "NUM_LINES": 17676013, "MD5": "c70da2ba79db33fb0fc9119cbad16260", "URL": "http://www.statmt.org/wmt11/training-monolingual-news-2010.tgz", "first_line": "I Need to Vent.\n"}
 {"dataset_name": "SQuAD1", "split": "train", "NUM_LINES": 87599, "MD5": {"train": "981b29407e0affa3b1b156f72073b945", "dev": "3e85deb501d4e538b6bc56f786231552"}, "URL": {"train": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json", "dev": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json"}, "first_line": ["Architecturally, the school has a Catholic character. Atop the Main Building's gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.", "To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France?", ["Saint Bernadette Soubirous"], [515]]}
 {"dataset_name": "SQuAD1", "split": "dev", "NUM_LINES": 10570, "MD5": {"train": "981b29407e0affa3b1b156f72073b945", "dev": "3e85deb501d4e538b6bc56f786231552"}, "URL": {"train": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json", "dev": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json"}, "first_line": ["Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24\u201310 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the \"golden anniversary\" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as \"Super Bowl L\"), so that the logo could prominently feature the Arabic numerals 50.", "Which NFL team represented the AFC at Super Bowl 50?", ["Denver Broncos", "Denver Broncos", "Denver Broncos"], [177, 177, 177]]}
 {"dataset_name": "SQuAD2", "split": "train", "NUM_LINES": 130319, "MD5": {"train": "62108c273c268d70893182d5cf8df740", "dev": "246adae8b7002f8679c027697b0b7cf8"}, "URL": {"train": "https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json", "dev": "https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json"}, "first_line": ["Beyonc\u00e9 Giselle Knowles-Carter (/bi\u02d0\u02c8j\u0252nse\u026a/ bee-YON-say) (born September 4, 1981) is an American singer, songwriter, record producer and actress. Born and raised in Houston, Texas, she performed in various singing and dancing competitions as a child, and rose to fame in the late 1990s as lead singer of R&B girl-group Destiny's Child. Managed by her father, Mathew Knowles, the group became one of the world's best-selling girl groups of all time. Their hiatus saw the release of Beyonc\u00e9's debut album, Dangerously in Love (2003), which established her as a solo artist worldwide, earned five Grammy Awards and featured the Billboard Hot 100 number-one singles \"Crazy in Love\" and \"Baby Boy\".", "When did Beyonce start becoming popular?", ["in the late 1990s"], [269]]}

test/data/test_builtin_datasets.py

@@ -173,9 +173,6 @@ def test_raw_text_classification(self, info):
         if dataset_name in GOOGLE_DRIVE_BASED_DATASETS:
             return
 
-        # Currently disabled due to incredibly slow download
-        if dataset_name == "WMTNewsCrawl":
-            return
         split = info['split']
         data_iter = torchtext.datasets.DATASETS[dataset_name](split=split)
         self.assertEqual(len(data_iter), info['NUM_LINES'])

torchtext/datasets/__init__.py

@@ -17,7 +17,6 @@
 from .wikitext103 import WikiText103
 from .wikitext2 import WikiText2
 from .wmt14 import WMT14
-from .wmtnewscrawl import WMTNewsCrawl
 from .yahooanswers import YahooAnswers
 from .yelpreviewfull import YelpReviewFull
 from .yelpreviewpolarity import YelpReviewPolarity
@@ -39,7 +38,6 @@
     'SogouNews': SogouNews,
     'UDPOS': UDPOS,
     'WMT14': WMT14,
-    'WMTNewsCrawl': WMTNewsCrawl,
     'WikiText103': WikiText103,
     'WikiText2': WikiText2,
     'YahooAnswers': YahooAnswers,

torchtext/experimental/datasets/language_modeling.py

@@ -3,6 +3,7 @@
 from torchtext.data.utils import get_tokenizer
 from torchtext.vocab import build_vocab_from_iterator
 from torchtext import datasets as raw
+from torchtext.experimental.datasets import raw as experimental_raw
 from torchtext.data.datasets_utils import check_default_set
 from torchtext.data.datasets_utils import wrap_datasets
 
@@ -69,7 +70,8 @@ def _setup_datasets(dataset_name, tokenizer, root, vocab, split_, year, language
         if 'train' not in split:
             raise TypeError("Must pass a vocab if train is not selected.")
         if dataset_name == 'WMTNewsCrawl':
-            raw_train, = raw.DATASETS[dataset_name](root=root, split=('train',), year=year, language=language)
+            raw_train = experimental_raw.DATASETS[dataset_name](root=root, split='train',
+                                                                year=year, language=language)
         else:
             raw_train, = raw.DATASETS[dataset_name](root=root, split=('train',))
         logger_.info('Building Vocab based on train data')
@@ -80,7 +82,8 @@ def text_transform(line):
         return torch.tensor([vocab[token] for token in tokenizer(line)], dtype=torch.long)
 
     if dataset_name == 'WMTNewsCrawl':
-        raw_datasets = raw.DATASETS[dataset_name](root=root, split=split, year=year, language=language)
+        raw_datasets = experimental_raw.DATASETS[dataset_name](root=root, split=split,
+                                                               year=year, language=language)
     else:
         raw_datasets = raw.DATASETS[dataset_name](root=root, split=split)
     raw_data = {name: list(map(text_transform, raw_dataset)) for name, raw_dataset in zip(split, raw_datasets)}
@@ -193,7 +196,7 @@ def PennTreebank(tokenizer=None, root='.data', vocab=None, split=('train', 'vali
     return _setup_datasets("PennTreebank", tokenizer, root, vocab, split, None, None)
 
 
-def WMTNewsCrawl(tokenizer=None, root='.data', vocab=None, split=('train'), year=2010, language='en'):
+def WMTNewsCrawl(tokenizer=None, root='.data', vocab=None, split='train', year=2010, language='en'):
     """ Defines WMTNewsCrawl datasets.
 
     Create language modeling dataset: WMTNewsCrawl
@@ -208,7 +211,7 @@ def WMTNewsCrawl(tokenizer=None, root='.data', vocab=None, split=('train'), year
         vocab: Vocabulary used for dataset. If None, it will generate a new
             vocabulary based on the train data set.
         split: a string or tuple for the returned datasets
-            (Default: ('train',))
+            (Default: 'train')
         year: the year of the dataset (Default: 2010)
         language: the language of the dataset (Default: 'en')
 

torchtext/experimental/datasets/raw/__init__.py

@@ -0,0 +1,18 @@
+import importlib
+from .wmtnewscrawl import WMTNewsCrawl
+
+DATASETS = {
+    'WMTNewsCrawl': WMTNewsCrawl,
+}
+
+URLS = {}
+NUM_LINES = {}
+MD5 = {}
+for dataset in DATASETS:
+    dataset_module_path = "torchtext.experimental.datasets.raw." + dataset.lower()
+    dataset_module = importlib.import_module(dataset_module_path)
+    URLS[dataset] = dataset_module.URL
+    NUM_LINES[dataset] = dataset_module.NUM_LINES
+    MD5[dataset] = dataset_module.MD5
+
+__all__ = sorted(list(map(str, DATASETS.keys())))

torchtext/datasets/wmtnewscrawl.py → torchtext/experimental/datasets/raw/wmtnewscrawl.py

@@ -42,7 +42,7 @@
 
 
 @add_docstring_header(num_lines=NUM_LINES)
-@wrap_split_argument(('train',))
+@wrap_split_argument('train')
 def WMTNewsCrawl(root, split, year=2010, language='en'):
     if year not in _AVAILABLE_YEARS:
         raise ValueError("{} not available. Please choose from years {}".format(year, _AVAILABLE_YEARS))