From 3e5759b68014db7f98b5c6b46c4ee90106b4be18 Mon Sep 17 00:00:00 2001 From: Enno Hermann Date: Wed, 18 Dec 2024 16:38:27 +0100 Subject: [PATCH] fix(notebooks): update plot_embeddings_umap notebook --- ...riTTS.ipynb => plot_embeddings_umap.ipynb} | 200 +++++++++--------- pyproject.toml | 2 +- 2 files changed, 104 insertions(+), 98 deletions(-) rename notebooks/{PlotUmapLibriTTS.ipynb => plot_embeddings_umap.ipynb} (56%) diff --git a/notebooks/PlotUmapLibriTTS.ipynb b/notebooks/plot_embeddings_umap.ipynb similarity index 56% rename from notebooks/PlotUmapLibriTTS.ipynb rename to notebooks/plot_embeddings_umap.ipynb index 1e29790b9e..b661f85673 100644 --- a/notebooks/PlotUmapLibriTTS.ipynb +++ b/notebooks/plot_embeddings_umap.ipynb @@ -4,13 +4,20 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Overview\n", + "# Overview\n", "\n", "This notebook can be used with both a single or multi- speaker corpus and allows the interactive plotting of speaker embeddings linked to underlying audio (see instructions in the repo's speaker_embedding directory)\n", "\n", "Depending on the directory structure used for your corpus, you may need to adjust handling of **speaker_to_utter** and **locations**." ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Imports\n" + ] + }, { "cell_type": "code", "execution_count": null, @@ -19,63 +26,47 @@ "source": [ "import os\n", "import glob\n", + "import random\n", + "from collections import defaultdict\n", + "from pathlib import Path\n", + "\n", "import numpy as np\n", + "import torch\n", "import umap\n", "\n", - "from TTS.utils.audio import AudioProcessor\n", + "from TTS.bin.compute_embeddings import compute_embeddings\n", "from TTS.config import load_config\n", + "from TTS.config.shared_configs import BaseDatasetConfig\n", + "from TTS.tts.datasets import load_tts_samples\n", + "from TTS.utils.audio import AudioProcessor\n", "\n", "from bokeh.io import output_notebook, show\n", "from bokeh.plotting import figure\n", "from bokeh.models import HoverTool, ColumnDataSource, BoxZoomTool, ResetTool, OpenURL, TapTool\n", "from bokeh.transform import factor_cmap\n", - "from bokeh.palettes import Category10" + "from bokeh.palettes import Category10\n", + "\n", + "output_notebook()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "For larger sets of speakers, you can use **Category20**, but you need to change it in the **pal** variable too\n", + "For larger sets of speakers, you can use `Category20`, but you need to change it in the `pal` variable too\n", "\n", - "List of Bokeh palettes here: http://docs.bokeh.org/en/1.4.0/docs/reference/palettes.html\n", + "List of Bokeh palettes here: https://docs.bokeh.org/en/latest/docs/reference/palettes.html\n", "\n", "**NB:** if you have problems with other palettes, first see https://stackoverflow.com/questions/48333820/why-do-some-bokeh-palettes-raise-a-valueerror-when-used-in-factor-cmap" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "output_notebook()" - ] - }, { "cell_type": "markdown", "metadata": {}, "source": [ - "You should also adjust all the path constants to point at the relevant locations for you locally" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "MODEL_RUN_PATH = \"/media/erogol/data_ssd/Models/libri_tts/speaker_encoder/libritts_360-half-October-31-2019_04+54PM-19d2f5f/\"\n", - "MODEL_PATH = MODEL_RUN_PATH + \"best_model.pth\"\n", - "CONFIG_PATH = MODEL_RUN_PATH + \"config.json\"\n", - "\n", - "# My single speaker locations\n", - "#EMBED_PATH = \"/home/neil/main/Projects/TTS3/embeddings/neil14/\"\n", - "#AUDIO_PATH = \"/home/neil/data/Projects/NeilTTS/neil14/wavs/\"\n", + "## Config\n", "\n", - "# My multi speaker locations\n", - "EMBED_PATH = \"/home/erogol/Data/Libri-TTS/train-clean-360-embed_128/\"\n", - "AUDIO_PATH = \"/home/erogol/Data/Libri-TTS/train-clean-360/\"" + "You should adjust all the paths to point at the relevant locations for you locally." ] }, { @@ -84,7 +75,16 @@ "metadata": {}, "outputs": [], "source": [ - "!ls -1 $MODEL_RUN_PATH" + "# Dataset\n", + "formatter_name = \"ljspeech\"\n", + "dataset_name = \"ljspeech\"\n", + "dataset_path = \"path/to/LJSpeech-1.1\"\n", + "meta_file_train = \"metadata.csv\"\n", + "\n", + "# Speaker encoder\n", + "se_model_path = \"https://github.com/coqui-ai/TTS/releases/download/speaker_encoder_model/model_se.pth.tar\"\n", + "se_config_path = \"https://github.com/coqui-ai/TTS/releases/download/speaker_encoder_model/config_se.json\"\n", + "embedding_path = \"speakers.pth\"" ] }, { @@ -93,15 +93,25 @@ "metadata": {}, "outputs": [], "source": [ - "CONFIG = load_config(CONFIG_PATH)\n", - "ap = AudioProcessor(**CONFIG['audio'])" + "dataset_config = BaseDatasetConfig()\n", + "dataset_config.formatter = formatter_name\n", + "dataset_config.dataset_name = dataset_name\n", + "dataset_config.path = dataset_path\n", + "dataset_config.meta_file_train = meta_file_train\n", + "\n", + "meta_data_train, meta_data_eval = load_tts_samples(dataset_config, eval_split=False)\n", + "utt_to_wav = {\n", + " item[\"audio_unique_name\"]: str(Path(item[\"audio_file\"]).relative_to(dataset_path)) for item in meta_data_train\n", + "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Bring in the embeddings created by **compute_embeddings.py**" + "## Compute embeddings\n", + "\n", + "You can skip this if you have already computed embeddings with `TTS/bin/compute_embeddings.py`" ] }, { @@ -110,33 +120,38 @@ "metadata": {}, "outputs": [], "source": [ - "embed_files = glob.glob(EMBED_PATH+\"/**/*.npy\", recursive=True)\n", - "print(f'Embeddings found: {len(embed_files)}')" + "compute_embeddings(\n", + " model_path=se_model_path,\n", + " config_path=se_config_path,\n", + " output_path=embedding_path,\n", + " formatter_name=formatter_name,\n", + " dataset_name=dataset_name,\n", + " dataset_path=dataset_path,\n", + " meta_file_train=meta_file_train,\n", + ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Check that we did indeed find an embedding" + "## Plot Umap" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "embed_files[0]" + "Bring in the embeddings created by `TTS/bin/compute_embeddings.py`" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "### Process the speakers\n", - "\n", - "Assumes count of **speaker_paths** corresponds to number of speakers (so a corpus in just one directory would be treated like a single speaker and the multiple directories of LibriTTS are treated as distinct speakers)" + "embeddings = torch.load(embedding_path, weights_only=True)" ] }, { @@ -145,15 +160,13 @@ "metadata": {}, "outputs": [], "source": [ - "speaker_paths = list(set([os.path.dirname(os.path.dirname(embed_file)) for embed_file in embed_files]))\n", - "speaker_to_utter = {}\n", - "for embed_file in embed_files:\n", - " speaker_path = os.path.dirname(os.path.dirname(embed_file))\n", - " try:\n", - " speaker_to_utter[speaker_path].append(embed_file)\n", - " except:\n", - " speaker_to_utter[speaker_path]=[embed_file]\n", - "print(f'Speaker count: {len(speaker_paths)}')" + "speakers = set()\n", + "speaker_to_utter = defaultdict(list)\n", + "for idx, embedding in embeddings.items():\n", + " speaker = embedding[\"name\"]\n", + " speakers.add(speaker)\n", + " speaker_to_utter[speaker].append(idx)\n", + "print(f\"Speaker count: {len(speakers)}\")" ] }, { @@ -175,35 +188,32 @@ "labels = []\n", "locations = []\n", "\n", - "# single speaker \n", - "#num_speakers = 1\n", - "#num_utters = 1000\n", + "# single speaker\n", + "num_speakers = 1\n", + "num_utters = 1000\n", "\n", "# multi speaker\n", - "num_speakers = 10\n", - "num_utters = 20\n", + "# num_speakers = 10\n", + "# num_utters = 20\n", "\n", - "\n", - "speaker_idxs = np.random.choice(range(len(speaker_paths)), num_speakers, replace=False )\n", + "speaker_idxs = random.sample(list(speakers), num_speakers)\n", "\n", "for speaker_num, speaker_idx in enumerate(speaker_idxs):\n", - " speaker_path = speaker_paths[speaker_idx]\n", - " speakers_utter = speaker_to_utter[speaker_path]\n", - " utter_idxs = np.random.randint(0, len(speakers_utter) , num_utters)\n", + " speakers_utter = speaker_to_utter[speaker_idx]\n", + " utter_idxs = random.sample(speakers_utter, num_utters)\n", " for utter_idx in utter_idxs:\n", - " embed_path = speaker_to_utter[speaker_path][utter_idx]\n", - " embed = np.load(embed_path)\n", - " embeds.append(embed)\n", - " labels.append(str(speaker_num))\n", - " locations.append(embed_path.replace(EMBED_PATH, '').replace('.npy','.wav'))\n", - "embeds = np.concatenate(embeds)" + " embed = np.array(embeddings[utter_idx][\"embedding\"])\n", + " embeds.append(embed)\n", + " labels.append(speaker_idx)\n", + " locations.append(utt_to_wav[utter_idx])\n", + "embeds = np.stack(embeds)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Load embeddings with UMAP" + "### Load embeddings with UMAP" ] }, { @@ -222,9 +232,7 @@ "source": [ "### Interactively charting the data in Bokeh\n", "\n", - "Set up various details for Bokeh to plot the data\n", - "\n", - "You can use the regular Bokeh [tools](http://docs.bokeh.org/en/1.4.0/docs/user_guide/tools.html?highlight=tools) to explore the data, with reset setting it back to normal\n", + "You can use the regular Bokeh [tools](https://docs.bokeh.org/en/latest/docs/user_guide/interaction/tools.html) to explore the data, with reset setting it back to normal\n", "\n", "Once you have started the local server (see cell below) you can then click on plotted points which will open a tab to play the audio for that point, enabling easy exploration of your corpus\n", "\n", @@ -238,22 +246,17 @@ "outputs": [], "source": [ "source_wav_stems = ColumnDataSource(\n", - " data=dict(\n", - " x = projection.T[0].tolist(),\n", - " y = projection.T[1].tolist(),\n", - " desc=locations,\n", - " label=labels\n", - " )\n", + " data=dict(\n", + " x=projection.T[0].tolist(),\n", + " y=projection.T[1].tolist(),\n", + " desc=locations,\n", + " label=labels,\n", " )\n", + ")\n", "\n", - "hover = HoverTool(\n", - " tooltips=[\n", - " (\"file\", \"@desc\"),\n", - " (\"speaker\", \"@label\"),\n", - " ]\n", - " )\n", + "hover = HoverTool(tooltips=[(\"file\", \"@desc\"), (\"speaker\", \"@label\")])\n", "\n", - "# optionally consider adding these to the tooltips if you want additional detail\n", + "### Optionally consider adding these to the tooltips if you want additional detail\n", "# for the coordinates: (\"(x,y)\", \"($x, $y)\"),\n", "# for the index of the embedding / wav file: (\"index\", \"$index\"),\n", "\n", @@ -261,10 +264,13 @@ "pal_size = max(len(factors), 3)\n", "pal = Category10[pal_size]\n", "\n", - "p = figure(plot_width=600, plot_height=400, tools=[hover,BoxZoomTool(), ResetTool(), TapTool()])\n", - "\n", - "\n", - "p.circle('x', 'y', source=source_wav_stems, color=factor_cmap('label', palette=pal, factors=factors),)\n", + "p = figure(width=600, height=400, tools=[hover, BoxZoomTool(), ResetTool(), TapTool()])\n", + "p.scatter(\n", + " \"x\",\n", + " \"y\",\n", + " source=source_wav_stems,\n", + " color=factor_cmap(\"label\", palette=pal, factors=factors),\n", + ")\n", "\n", "url = \"http://localhost:8000/@desc\"\n", "taptool = p.select(type=TapTool)\n", @@ -292,7 +298,7 @@ "metadata": {}, "outputs": [], "source": [ - "%cd $AUDIO_PATH\n", + "%cd $dataset_path\n", "%pwd\n", "!python -m http.server" ] @@ -300,7 +306,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -314,7 +320,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.4" + "version": "3.10.14" } }, "nbformat": 4, diff --git a/pyproject.toml b/pyproject.toml index 16d990c169..b27e1cd4d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,7 +93,7 @@ dependencies = [ [project.optional-dependencies] # Only used in notebooks notebooks = [ - "bokeh==1.4.0", + "bokeh>=3.0.3", "pandas>=1.4,<2.0", "umap-learn>=0.5.1", ]