tsvitsfe.py

import torch
import os
import json
import math
import torch
from torch import nn
from torch.nn import functional as F

import commons
import utils
from text import text_to_sequence
from bertfe import BERTFrontEnd
from moji.moji import TorchMoji
from stft import TorchSTFT
from text.cleaners import arpa_cleaners
from text import cleaned_text_to_sequence


class TSVITSFE:
  # device can be cpu or cuda
  def load(self, model_path, config_path, device, bert_model_name = "huawei-noah/TinyBERT_General_4L_312D"):
    self.model = torch.jit.load(model_path).to(device)
    self.moji = TorchMoji(verbose=True)
    self.bert_f = BERTFrontEnd("cuda" in device, model_name=bert_model_name)
    self.hps = utils.get_hparams_from_file(config_path)
    self.stft = TorchSTFT(self.hps.model.gen_istft_n_fft, hop_length= self.hps.model.gen_istft_hop_size,
                        win_length=self.hps.model.gen_istft_n_fft).to(device)
    self.device = device

  
 

  def get_arpa_text(self, text):
    text_pre = arpa_cleaners(text)
    text_norm = cleaned_text_to_sequence(text_pre)
    if self.hps.data.add_blank:
      text_norm = commons.intersperse(text_norm, 0)
    text_norm = torch.LongTensor(text_norm)
    return text_norm
 
  def get_text(self, text):
    text_norm = text_to_sequence(text, self.hps.data.text_cleaners)
    if self.hps.data.add_blank:
        text_norm = commons.intersperse(text_norm, 0)
    text_norm = torch.LongTensor(text_norm)
    return text_norm
  
  # Inference
  # in_em = Input to the emotion predictor. Default empty, where it will use same as text
  def infer(self,in_text, in_em = "", in_speed = 1.):
  
    if "arpa_cleaners" in self.hps.data.text_cleaners:
      stn_tst = self.get_arpa_text(in_text)
    else:
      stn_tst = self.get_text(in_text)
    
    if len(in_em) == 0:
      in_em = in_text

    with torch.no_grad():
      x_tst = stn_tst.unsqueeze(0).to(self.device)
      x_tst_lengths = torch.LongTensor([stn_tst.size(0)]).to(self.device)
      len_scale = torch.FloatTensor([in_speed]).to(self.device)
      em = self.moji(in_em).squeeze().unsqueeze(0).to(self.device) # [1, 1, 2304] -> [2304] -> [1, 2304]
        
      bert_emb, _ = self.bert_f.infer(in_text) # [1, tokens, channels]
      bert_emb = bert_emb.to(self.device)
      bert_lens = torch.LongTensor([bert_emb.size(1)]).to(self.device)

      spec, phase, x1, attn = self.model.infer_ts_noistft(x_tst, x_tst_lengths, em, bert_emb, bert_lens, len_scale)
      audio = self.stft.inverse(spec, phase)
      audio = audio.squeeze().cpu().data.float().numpy()
    
    return audio