Upgrade readme to reflect new tokenizers API

README.md

@@ -31,10 +31,17 @@ The sentence piece model loads the same `.model` proto files as the [C++ library
 # Usage example (Rust)
 
 ```rust
-let vocab = Arc::new(rust_tokenizers::BertVocab::from_file(&vocab_path));
+use std::path::PathBuf;
 
-let test_sentence = Example::new_from_string("This is a sample sentence to be tokenized");
-let bert_tokenizer: BertTokenizer = BertTokenizer::from_existing_vocab(vocab.clone());
+use rust_tokenizers::tokenizer::{BertTokenizer, Tokenizer, TruncationStrategy};
+use rust_tokenizers::vocab::{BertVocab, Vocab};
+
+let lowercase: bool = true;
+let strip_accents: bool = true;
+let vocab_path: PathBuf  = PathBuf::from("path/to/vocab");
+let vocab: BertVocab = BertVocab::from_file(&vocab_path)?;
+let test_sentence: Example = Example::new_from_string("This is a sample sentence to be tokenized");
+let bert_tokenizer: BertTokenizer = BertTokenizer::from_existing_vocab(vocab, lowercase, strip_accents);
 
 println!("{:?}", bert_tokenizer.encode(&test_sentence.sentence_1,
                                        None,

main/README.md

@@ -25,10 +25,17 @@ Using the tokenizers requires downloading manually the tokenizers required files
 # Usage example
 
 ```rust
-let vocab = Arc::new(rust_tokenizers::BertVocab::from_file(&vocab_path));
+use std::path::PathBuf;
 
-let test_sentence = Example::new_from_string("This is a sample sentence to be tokenized");
-let bert_tokenizer: BertTokenizer = BertTokenizer::from_existing_vocab(vocab.clone());
+use rust_tokenizers::tokenizer::{BertTokenizer, Tokenizer, TruncationStrategy};
+use rust_tokenizers::vocab::{BertVocab, Vocab};
+
+let lowercase: bool = true;
+let strip_accents: bool = true;
+let vocab_path: PathBuf  = PathBuf::from("path/to/vocab");
+let vocab: BertVocab = BertVocab::from_file(&vocab_path)?;
+let test_sentence: Example = Example::new_from_string("This is a sample sentence to be tokenized");
+let bert_tokenizer: BertTokenizer = BertTokenizer::from_existing_vocab(vocab, lowercase, strip_accents);
 
 println!("{:?}", bert_tokenizer.encode(&test_sentence.sentence_1,
                                        None,

main/src/lib.rs

@@ -47,9 +47,11 @@
 //! use rust_tokenizers::vocab::{BertVocab, Vocab};
 //! let vocab_path = "path/to/vocab";
 //! let vocab = BertVocab::from_file(&vocab_path)?;
+//! let lowercase: bool = true;
+//! let strip_accents: bool = true;
 //!
 //! let test_sentence = Example::new_from_string("This is a sample sentence to be tokenized");
-//! let bert_tokenizer: BertTokenizer = BertTokenizer::from_existing_vocab(vocab, true, true);
+//! let bert_tokenizer: BertTokenizer = BertTokenizer::from_existing_vocab(vocab, lowercase, strip_accents);
 //!
 //! println!(
 //!     "{:?}",