Swapping experimental Vocab and retiring current Vocab into legacy (#…

…1289) Summary: allow-large-files to commit wikitext103_vocab.pt Reviewed By: cpuhrsch Differential Revision: D28478152 fbshipit-source-id: c2a871439f054024b95c05f7664a84028aacaca3
pytorch · May 19, 2021 · c8bced1 · c8bced1
1 parent 6231993
commit c8bced1
Show file tree

Hide file tree

Showing 26 changed files with 842 additions and 856 deletions.
diff --git a/benchmark/benchmark_experimental_vocab.py b/benchmark/benchmark_experimental_vocab.py
@@ -12,7 +12,7 @@
  load_vocab_from_file,
  build_vocab_from_text_file
 )
-from torchtext.vocab import (
+from torchtext.legacy.vocab import (
  Vocab,
  build_vocab_from_iterator
 )

diff --git a/docs/source/experimental_vocab.rst b/docs/source/experimental_vocab.rst
@@ -7,13 +7,6 @@ torchtext.experimental.vocab
 .. automodule:: torchtext.experimental.vocab
 .. currentmodule:: torchtext.experimental.vocab
 
-:hidden:`Vocab`
-~~~~~~~~~~~~~~~
-
-.. autoclass:: Vocab
- :members:
- :special-members:
-
 :hidden:`vocab`
 ~~~~~~~~~~~~~~~
 

diff --git a/docs/source/vocab.rst b/docs/source/vocab.rst
@@ -12,14 +12,8 @@ torchtext.vocab
 
 .. autoclass:: Vocab
  :members:
- :special-members: __init__
-
-:hidden:`SubwordVocab`
-~~~~~~~~~~~~~~~~~~~~~~
+ :special-members:
 
-.. autoclass:: SubwordVocab
- :members:
- :special-members: __init__
 
 :hidden:`Vectors`
 ~~~~~~~~~~~~~~~~~
@@ -48,12 +42,3 @@ Pretrained Word Embeddings
 
 .. autoclass:: CharNGram
  :members:
-
-Misc.
------
-
-:hidden:`build_vocab_from_iterator`
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. autofunction:: build_vocab_from_iterator
-
diff --git a/examples/BERT/mlm_task.py b/examples/BERT/mlm_task.py
@@ -125,7 +125,7 @@ def run_main(args, rank=None):
  except:
  train_dataset, valid_dataset, test_dataset = WLMDataset()
  old_vocab = train_dataset.vocab
- vocab = torchtext.vocab.Vocab(counter=old_vocab.freqs,
+ vocab = torchtext.legacy.vocab.Vocab(counter=old_vocab.freqs,
  specials=['<unk>', '<pad>', '<MASK>'])
  with open(args.save_vocab, 'wb') as f:
  torch.save(vocab, f)

diff --git a/examples/BERT/qa_task.py b/examples/BERT/qa_task.py
@@ -163,7 +163,7 @@ def train():
  except:
  train_dataset, dev_dataset = SQuAD1()
  old_vocab = train_dataset.vocab
- vocab = torchtext.vocab.Vocab(counter=old_vocab.freqs,
+ vocab = torchtext.legacy.vocab.Vocab(counter=old_vocab.freqs,
  specials=['<unk>', '<pad>', '<MASK>'])
  with open(args.save_vocab, 'wb') as f:
  torch.save(vocab, f)

diff --git a/examples/data_pipeline/pipelines.py b/examples/data_pipeline/pipelines.py
@@ -45,7 +45,7 @@ def build_sp_pipeline(args):
 def build_legacy_torchtext_vocab_pipeline(args):
  vocab_file = args.vocab_filename
  tokenizer = get_tokenizer("basic_english")
- from torchtext.vocab import build_vocab_from_iterator
+ from torchtext.legacy.vocab import build_vocab_from_iterator
 
  def token_iterator(vocab_file):
  f = open(vocab_file, 'r')
@@ -72,7 +72,7 @@ def build_experimental_torchtext_pipeline(args):
 def build_legacy_batch_torchtext_vocab_pipeline(args):
  vocab_file = args.vocab_filename
  tokenizer = get_tokenizer("basic_english")
- from torchtext.vocab import build_vocab_from_iterator
+ from torchtext.legacy.vocab import build_vocab_from_iterator
 
  def token_iterator(vocab_file):
  f = open(vocab_file, 'r')

diff --git a/examples/vocab/pytext_vocab.py b/examples/vocab/pytext_vocab.py
@@ -2,7 +2,8 @@
 
 from fairseq.data.dictionary import Dictionary
 import torch
-from torchtext.experimental.vocab import vocab, Vocab
+from torchtext.experimental.vocab import vocab
+from torchtext.vocab import Vocab
 from typing import Dict, List, Optional
 
 

diff --git a/examples/vocab/vocab.py b/examples/vocab/vocab.py
@@ -4,7 +4,7 @@
 import torch
 import io
 
-from torchtext.vocab import build_vocab_from_iterator
+from torchtext.legacy.vocab import build_vocab_from_iterator
 from torchtext.data.utils import ngrams_iterator
 from torchtext.data.utils import get_tokenizer
 from torchtext.utils import unicode_csv_reader

diff --git a/test/asset/wikitext103_vocab.pt b/test/asset/wikitext103_vocab.pt
diff --git a/test/data/test_builtin_datasets.py b/test/data/test_builtin_datasets.py
@@ -210,7 +210,7 @@ def test_next_method_dataset(self):
 
  def test_imdb(self):
  from torchtext.experimental.datasets import IMDB
- from torchtext.vocab import Vocab
+ from torchtext.legacy.vocab import Vocab
  # smoke test to ensure imdb works properly
  train_dataset, test_dataset = IMDB()
  self._helper_test_func(len(train_dataset), 25000, train_dataset[0][1][:10],
@@ -465,7 +465,7 @@ def test_conll_sequence_tagging(self):
 
  def test_squad1(self):
  from torchtext.experimental.datasets import SQuAD1
- from torchtext.vocab import Vocab
+ from torchtext.legacy.vocab import Vocab
  # smoke test to ensure imdb works properly
  train_dataset, dev_dataset = SQuAD1()
  context, question, answers, ans_pos = train_dataset[100]
@@ -494,7 +494,7 @@ def test_squad1(self):
 
  def test_squad2(self):
  from torchtext.experimental.datasets import SQuAD2
- from torchtext.vocab import Vocab
+ from torchtext.legacy.vocab import Vocab
  # smoke test to ensure imdb works properly
  train_dataset, dev_dataset = SQuAD2()
  context, question, answers, ans_pos = train_dataset[200]

diff --git a/test/experimental/test_vocab.py b/test/experimental/test_vocab.py