Resume for the SVC's vocalist pretrained ckpt

.gitignore

@@ -33,6 +33,7 @@ egs/svc/custom
 egs/svc/*/dev*
 egs/svc/dev_exp_config.json
 bins/svc/demo*
+bins/svc/preprocess_custom.py
 data
 ckpts
 

models/svc/base/svc_dataset.py

@@ -164,14 +164,9 @@ def __init__(self, args, cfg, infer_type):
         self.trans_key = args.trans_key
         assert type(target_singer) == str
 
-        # self.target_singer = target_singer.split("_")[-1]
-        # self.target_dataset = target_singer.replace(
-        #     "_{}".format(self.target_singer), ""
-        # )
-
-        self.target_dataset = target_singer.split("_")[0]
-        self.target_singer = target_singer.replace(
-            "{}_".format(self.target_dataset), ""
+        self.target_singer = target_singer.split("_")[-1]
+        self.target_dataset = target_singer.replace(
+            "_{}".format(self.target_singer), ""
         )
 
         self.target_mel_extrema = load_mel_extrema(cfg.preprocess, self.target_dataset)

modules/encoder/condition_encoder.py

@@ -13,29 +13,34 @@
 
 
 class ContentEncoder(nn.Module):
-    def __init__(self, input_dim, output_dim):
+    def __init__(self, cfg, input_dim, output_dim):
         super().__init__()
-
-        # if input_dim != 0:
-        #     self.nn = nn.Linear(input_dim, output_dim)
+        self.cfg = cfg
 
         assert input_dim != 0
-
-        # TODO: introduce conformer
-        self.pos_encoder = PositionalEncoding(input_dim)
-        self.conformer = Conformer(
-            input_dim=input_dim,
-            num_heads=2,
-            ffn_dim=256,
-            num_layers=6,
-            depthwise_conv_kernel_size=3,
-        )
         self.nn = nn.Linear(input_dim, output_dim)
 
+        # Introduce conformer or not
+        if (
+            "use_conformer_for_content_features" in cfg
+            and cfg.use_conformer_for_content_features
+        ):
+            self.pos_encoder = PositionalEncoding(input_dim)
+            self.conformer = Conformer(
+                input_dim=input_dim,
+                num_heads=2,
+                ffn_dim=256,
+                num_layers=6,
+                depthwise_conv_kernel_size=3,
+            )
+        else:
+            self.conformer = None
+
     def forward(self, x, length=None):
         # x: (N, seq_len, input_dim) -> (N, seq_len, output_dim)
-        x = self.pos_encoder(x)
-        x, _ = self.conformer(x, length)
+        if self.conformer:
+            x = self.pos_encoder(x)
+            x, _ = self.conformer(x, length)
         return self.nn(x)
 
 
@@ -160,22 +165,22 @@ def __init__(self, cfg):
 
         if cfg.use_whisper:
             self.whisper_encoder = ContentEncoder(
-                self.cfg.whisper_dim, self.cfg.content_encoder_dim
+                self.cfg, self.cfg.whisper_dim, self.cfg.content_encoder_dim
             )
 
         if cfg.use_contentvec:
             self.contentvec_encoder = ContentEncoder(
-                self.cfg.contentvec_dim, self.cfg.content_encoder_dim
+                self.cfg, self.cfg.contentvec_dim, self.cfg.content_encoder_dim
             )
 
         if cfg.use_mert:
             self.mert_encoder = ContentEncoder(
-                self.cfg.mert_dim, self.cfg.content_encoder_dim
+                self.cfg, self.cfg.mert_dim, self.cfg.content_encoder_dim
             )
 
         if cfg.use_wenet:
             self.wenet_encoder = ContentEncoder(
-                self.cfg.wenet_dim, self.cfg.content_encoder_dim
+                self.cfg, self.cfg.wenet_dim, self.cfg.content_encoder_dim
             )
 
         self.melody_encoder = MelodyEncoder(self.cfg)

modules/whisper_extractor/__init__.py

@@ -36,7 +36,10 @@
 import torch
 from tqdm import tqdm
 
+from .audio import load_audio, log_mel_spectrogram, pad_or_trim
+from .decoding import DecodingOptions, DecodingResult, decode, detect_language
 from .model import Whisper, ModelDimensions
+from .transcribe import transcribe
 from .version import __version__