Skip to content

Commit

Permalink
fix(notebooks): update plot_embeddings_umap notebook
Browse files Browse the repository at this point in the history
  • Loading branch information
@eginhard
eginhard committed Dec 18, 2024
1 parent 8f8873c commit 3e5759b
Show file tree
Hide file tree
Showing 2 changed files with 104 additions and 98 deletions.
200 changes: 103 additions & 97 deletions notebooks/PlotUmapLibriTTS.ipynb → notebooks/plot_embeddings_umap.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,20 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"### Overview\n",
"# Overview\n",
"\n",
"This notebook can be used with both a single or multi- speaker corpus and allows the interactive plotting of speaker embeddings linked to underlying audio (see instructions in the repo's speaker_embedding directory)\n",
"\n",
"Depending on the directory structure used for your corpus, you may need to adjust handling of **speaker_to_utter** and **locations**."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Imports\n"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand All @@ -19,63 +26,47 @@
"source": [
"import os\n",
"import glob\n",
"import random\n",
"from collections import defaultdict\n",
"from pathlib import Path\n",
"\n",
"import numpy as np\n",
"import torch\n",
"import umap\n",
"\n",
"from TTS.utils.audio import AudioProcessor\n",
"from TTS.bin.compute_embeddings import compute_embeddings\n",
"from TTS.config import load_config\n",
"from TTS.config.shared_configs import BaseDatasetConfig\n",
"from TTS.tts.datasets import load_tts_samples\n",
"from TTS.utils.audio import AudioProcessor\n",
"\n",
"from bokeh.io import output_notebook, show\n",
"from bokeh.plotting import figure\n",
"from bokeh.models import HoverTool, ColumnDataSource, BoxZoomTool, ResetTool, OpenURL, TapTool\n",
"from bokeh.transform import factor_cmap\n",
"from bokeh.palettes import Category10"
"from bokeh.palettes import Category10\n",
"\n",
"output_notebook()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"For larger sets of speakers, you can use **Category20**, but you need to change it in the **pal** variable too\n",
"For larger sets of speakers, you can use `Category20`, but you need to change it in the `pal` variable too\n",
"\n",
"List of Bokeh palettes here: http://docs.bokeh.org/en/1.4.0/docs/reference/palettes.html\n",
"List of Bokeh palettes here: https://docs.bokeh.org/en/latest/docs/reference/palettes.html\n",
"\n",
"**NB:** if you have problems with other palettes, first see https://stackoverflow.com/questions/48333820/why-do-some-bokeh-palettes-raise-a-valueerror-when-used-in-factor-cmap"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"output_notebook()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"You should also adjust all the path constants to point at the relevant locations for you locally"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"MODEL_RUN_PATH = \"/media/erogol/data_ssd/Models/libri_tts/speaker_encoder/libritts_360-half-October-31-2019_04+54PM-19d2f5f/\"\n",
"MODEL_PATH = MODEL_RUN_PATH + \"best_model.pth\"\n",
"CONFIG_PATH = MODEL_RUN_PATH + \"config.json\"\n",
"\n",
"# My single speaker locations\n",
"#EMBED_PATH = \"/home/neil/main/Projects/TTS3/embeddings/neil14/\"\n",
"#AUDIO_PATH = \"/home/neil/data/Projects/NeilTTS/neil14/wavs/\"\n",
"## Config\n",
"\n",
"# My multi speaker locations\n",
"EMBED_PATH = \"/home/erogol/Data/Libri-TTS/train-clean-360-embed_128/\"\n",
"AUDIO_PATH = \"/home/erogol/Data/Libri-TTS/train-clean-360/\""
"You should adjust all the paths to point at the relevant locations for you locally."
]
},
{
Expand All @@ -84,7 +75,16 @@
"metadata": {},
"outputs": [],
"source": [
"!ls -1 $MODEL_RUN_PATH"
"# Dataset\n",
"formatter_name = \"ljspeech\"\n",
"dataset_name = \"ljspeech\"\n",
"dataset_path = \"path/to/LJSpeech-1.1\"\n",
"meta_file_train = \"metadata.csv\"\n",
"\n",
"# Speaker encoder\n",
"se_model_path = \"https://github.com/coqui-ai/TTS/releases/download/speaker_encoder_model/model_se.pth.tar\"\n",
"se_config_path = \"https://github.com/coqui-ai/TTS/releases/download/speaker_encoder_model/config_se.json\"\n",
"embedding_path = \"speakers.pth\""
]
},
{
Expand All @@ -93,15 +93,25 @@
"metadata": {},
"outputs": [],
"source": [
"CONFIG = load_config(CONFIG_PATH)\n",
"ap = AudioProcessor(**CONFIG['audio'])"
"dataset_config = BaseDatasetConfig()\n",
"dataset_config.formatter = formatter_name\n",
"dataset_config.dataset_name = dataset_name\n",
"dataset_config.path = dataset_path\n",
"dataset_config.meta_file_train = meta_file_train\n",
"\n",
"meta_data_train, meta_data_eval = load_tts_samples(dataset_config, eval_split=False)\n",
"utt_to_wav = {\n",
" item[\"audio_unique_name\"]: str(Path(item[\"audio_file\"]).relative_to(dataset_path)) for item in meta_data_train\n",
"}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Bring in the embeddings created by **compute_embeddings.py**"
"## Compute embeddings\n",
"\n",
"You can skip this if you have already computed embeddings with `TTS/bin/compute_embeddings.py`"
]
},
{
Expand All @@ -110,33 +120,38 @@
"metadata": {},
"outputs": [],
"source": [
"embed_files = glob.glob(EMBED_PATH+\"/**/*.npy\", recursive=True)\n",
"print(f'Embeddings found: {len(embed_files)}')"
"compute_embeddings(\n",
" model_path=se_model_path,\n",
" config_path=se_config_path,\n",
" output_path=embedding_path,\n",
" formatter_name=formatter_name,\n",
" dataset_name=dataset_name,\n",
" dataset_path=dataset_path,\n",
" meta_file_train=meta_file_train,\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Check that we did indeed find an embedding"
"## Plot Umap"
]
},
{
"cell_type": "code",
"execution_count": null,
"cell_type": "markdown",
"metadata": {},
"outputs": [],
"source": [
"embed_files[0]"
"Bring in the embeddings created by `TTS/bin/compute_embeddings.py`"
]
},
{
"cell_type": "markdown",
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"### Process the speakers\n",
"\n",
"Assumes count of **speaker_paths** corresponds to number of speakers (so a corpus in just one directory would be treated like a single speaker and the multiple directories of LibriTTS are treated as distinct speakers)"
"embeddings = torch.load(embedding_path, weights_only=True)"
]
},
{
Expand All @@ -145,15 +160,13 @@
"metadata": {},
"outputs": [],
"source": [
"speaker_paths = list(set([os.path.dirname(os.path.dirname(embed_file)) for embed_file in embed_files]))\n",
"speaker_to_utter = {}\n",
"for embed_file in embed_files:\n",
" speaker_path = os.path.dirname(os.path.dirname(embed_file))\n",
" try:\n",
" speaker_to_utter[speaker_path].append(embed_file)\n",
" except:\n",
" speaker_to_utter[speaker_path]=[embed_file]\n",
"print(f'Speaker count: {len(speaker_paths)}')"
"speakers = set()\n",
"speaker_to_utter = defaultdict(list)\n",
"for idx, embedding in embeddings.items():\n",
" speaker = embedding[\"name\"]\n",
" speakers.add(speaker)\n",
" speaker_to_utter[speaker].append(idx)\n",
"print(f\"Speaker count: {len(speakers)}\")"
]
},
{
Expand All @@ -175,35 +188,32 @@
"labels = []\n",
"locations = []\n",
"\n",
"# single speaker \n",
"#num_speakers = 1\n",
"#num_utters = 1000\n",
"# single speaker\n",
"num_speakers = 1\n",
"num_utters = 1000\n",
"\n",
"# multi speaker\n",
"num_speakers = 10\n",
"num_utters = 20\n",
"# num_speakers = 10\n",
"# num_utters = 20\n",
"\n",
"\n",
"speaker_idxs = np.random.choice(range(len(speaker_paths)), num_speakers, replace=False )\n",
"speaker_idxs = random.sample(list(speakers), num_speakers)\n",
"\n",
"for speaker_num, speaker_idx in enumerate(speaker_idxs):\n",
" speaker_path = speaker_paths[speaker_idx]\n",
" speakers_utter = speaker_to_utter[speaker_path]\n",
" utter_idxs = np.random.randint(0, len(speakers_utter) , num_utters)\n",
" speakers_utter = speaker_to_utter[speaker_idx]\n",
" utter_idxs = random.sample(speakers_utter, num_utters)\n",
" for utter_idx in utter_idxs:\n",
" embed_path = speaker_to_utter[speaker_path][utter_idx]\n",
" embed = np.load(embed_path)\n",
" embeds.append(embed)\n",
" labels.append(str(speaker_num))\n",
" locations.append(embed_path.replace(EMBED_PATH, '').replace('.npy','.wav'))\n",
"embeds = np.concatenate(embeds)"
" embed = np.array(embeddings[utter_idx][\"embedding\"])\n",
" embeds.append(embed)\n",
" labels.append(speaker_idx)\n",
" locations.append(utt_to_wav[utter_idx])\n",
"embeds = np.stack(embeds)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Load embeddings with UMAP"
"### Load embeddings with UMAP"
]
},
{
Expand All @@ -222,9 +232,7 @@
"source": [
"### Interactively charting the data in Bokeh\n",
"\n",
"Set up various details for Bokeh to plot the data\n",
"\n",
"You can use the regular Bokeh [tools](http://docs.bokeh.org/en/1.4.0/docs/user_guide/tools.html?highlight=tools) to explore the data, with reset setting it back to normal\n",
"You can use the regular Bokeh [tools](https://docs.bokeh.org/en/latest/docs/user_guide/interaction/tools.html) to explore the data, with reset setting it back to normal\n",
"\n",
"Once you have started the local server (see cell below) you can then click on plotted points which will open a tab to play the audio for that point, enabling easy exploration of your corpus\n",
"\n",
Expand All @@ -238,33 +246,31 @@
"outputs": [],
"source": [
"source_wav_stems = ColumnDataSource(\n",
" data=dict(\n",
" x = projection.T[0].tolist(),\n",
" y = projection.T[1].tolist(),\n",
" desc=locations,\n",
" label=labels\n",
" )\n",
" data=dict(\n",
" x=projection.T[0].tolist(),\n",
" y=projection.T[1].tolist(),\n",
" desc=locations,\n",
" label=labels,\n",
" )\n",
")\n",
"\n",
"hover = HoverTool(\n",
" tooltips=[\n",
" (\"file\", \"@desc\"),\n",
" (\"speaker\", \"@label\"),\n",
" ]\n",
" )\n",
"hover = HoverTool(tooltips=[(\"file\", \"@desc\"), (\"speaker\", \"@label\")])\n",
"\n",
"# optionally consider adding these to the tooltips if you want additional detail\n",
"### Optionally consider adding these to the tooltips if you want additional detail\n",
"# for the coordinates: (\"(x,y)\", \"($x, $y)\"),\n",
"# for the index of the embedding / wav file: (\"index\", \"$index\"),\n",
"\n",
"factors = list(set(labels))\n",
"pal_size = max(len(factors), 3)\n",
"pal = Category10[pal_size]\n",
"\n",
"p = figure(plot_width=600, plot_height=400, tools=[hover,BoxZoomTool(), ResetTool(), TapTool()])\n",
"\n",
"\n",
"p.circle('x', 'y', source=source_wav_stems, color=factor_cmap('label', palette=pal, factors=factors),)\n",
"p = figure(width=600, height=400, tools=[hover, BoxZoomTool(), ResetTool(), TapTool()])\n",
"p.scatter(\n",
" \"x\",\n",
" \"y\",\n",
" source=source_wav_stems,\n",
" color=factor_cmap(\"label\", palette=pal, factors=factors),\n",
")\n",
"\n",
"url = \"http://localhost:8000/@desc\"\n",
"taptool = p.select(type=TapTool)\n",
Expand Down Expand Up @@ -292,15 +298,15 @@
"metadata": {},
"outputs": [],
"source": [
"%cd $AUDIO_PATH\n",
"%cd $dataset_path\n",
"%pwd\n",
"!python -m http.server"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
Expand All @@ -314,7 +320,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.4"
"version": "3.10.14"
}
},
"nbformat": 4,
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ dependencies = [
[project.optional-dependencies]
# Only used in notebooks
notebooks = [
"bokeh==1.4.0",
"bokeh>=3.0.3",
"pandas>=1.4,<2.0",
"umap-learn>=0.5.1",
]
Expand Down

0 comments on commit 3e5759b

Please sign in to comment.