prosody/funasr/mandarin_cer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Unfortunately Paraformer packages are not compatible with ESPnet. Run\n",
    "```\n",
    "conda create -n funasr python=3.10  # installing jiwer fails on 3.11 (levenshtein dependency)\n",
    "conda activate funasr\n",
    "pip install jiwer==2.5  # 2.6 onwards conflicts with funasr's g2p click dependency\n",
    "pip install -U modelscope==1.10 funasr==0.8.7 pywordseg torchaudio transformers==4.39.2 charset-normalizer tabulate overrides==4.1.2\n",
    "```\n",
    "before running this."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "source": [
    "from pathlib import Path\n",
    "import os\n",
    "from modelscope.utils.constant import Tasks\n",
    "\n",
    "PWD = %pwd\n",
    "PWD = Path(PWD)\n",
    "prosody_dir = PWD.parent\n",
    "outputs_dir = PWD / 'outputs'\n",
    "os.makedirs(outputs_dir, exist_ok=True)\n",
    "prosody_outdir = prosody_dir / 'outputs'\n",
    "jets_dir = prosody_outdir / 'tts_train_jets_raw_phn_tacotron_g2p_en_no_space/aishell3'\n",
    "nopitch_dir = prosody_outdir / 'tts_train_jets_raw_phn_tacotron_g2p_en_no_space/aishell3_nopitch'\n",
    "nodur_dir = prosody_outdir / 'tts_train_jets_raw_phn_tacotron_g2p_en_no_space/aishell3_nopitch_nodur'\n",
    "baseline_dir = prosody_outdir / 'zm-text-tts/aishell3'\n",
    "data_dir = (prosody_dir / '../../datasets/data_aishell3/').resolve()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-07-17T08:44:31.535996Z",
     "start_time": "2024-07-17T08:44:31.530228Z"
    }
   },
   "outputs": [],
   "execution_count": 82
  },
  {
   "cell_type": "code",
   "source": "transcript_file = data_dir / 'test/content_pinyin.txt'",
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-07-16T11:01:19.610643Z",
     "start_time": "2024-07-16T11:01:19.608446Z"
    }
   },
   "outputs": [],
   "execution_count": 2
  },
  {
   "cell_type": "code",
   "source": [
    "def get_transcripts():\n",
    "    transcripts = {}\n",
    "    with open(transcript_file) as f:\n",
    "        for line in f:\n",
    "            wav_file, transcript = line.strip().split('\\t', maxsplit=1)\n",
    "            hans, pinyin_str = transcript.split('|', maxsplit=1)\n",
    "            pinyin_str = ''.join(pinyin_str.split())\n",
    "            transcripts[wav_file] = (hans, pinyin_str)\n",
    "    return transcripts\n",
    "\n",
    "transcripts = get_transcripts()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-07-16T11:01:19.819558Z",
     "start_time": "2024-07-16T11:01:19.753724Z"
    }
   },
   "outputs": [],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-16T11:01:20.986476Z",
     "start_time": "2024-07-16T11:01:20.557011Z"
    }
   },
   "cell_type": "code",
   "source": "from prosody.pinyin import hans_to_pinyins",
   "outputs": [],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-16T15:56:42.524111Z",
     "start_time": "2024-07-16T15:56:39.004879Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from funasr import AutoModel\n",
    "# paraformer-zh is a multi-functional asr model\n",
    "# use vad, punc, spk or not as you need\n",
    "auto_model = AutoModel(\n",
    "    # model='paraformer-zh',\n",
    "    model='iic/speech_paraformer_asr_nat-zh-cn-16k-aishell2-vocab5212-pytorch', hub='ms',\n",
    "    # vad_model=\"fsmn-vad\", vad_model_revision=\"v2.0.4\",\n",
    "    # punc_model=\"ct-punc-c\", punc_model_revision=\"v2.0.4\",\n",
    "    # spk_model=\"cam++\", spk_model_revision=\"v2.0.2\",\n",
    ")\n",
    "model, kwargs = auto_model.model, auto_model.kwargs\n",
    "kwargs['frontend'].fs = 22050\n",
    "asr_dir = outputs_dir / 'aishell3'\n",
    "os.makedirs(asr_dir, exist_ok=True)\n",
    "asr_dir = asr_dir / 'aishell2-vocab5212'\n",
    "os.makedirs(asr_dir, exist_ok=True)"
   ],
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-07-16 23:56:41,066 - modelscope - WARNING - Using the master branch is fragile, please use it with caution!\n",
      "2024-07-16 23:56:41,067 - modelscope - INFO - Use user-specified model revision: master\n",
      "WARNING:root:Using legacy_rel_pos and it will be deprecated in the future.\n",
      "WARNING:root:Using legacy_rel_selfattn and it will be deprecated in the future.\n"
     ]
    }
   ],
   "execution_count": 57
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-16T00:33:16.112652Z",
     "start_time": "2024-07-16T00:33:15.930425Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from funasr.register import tables\n",
    "from pprint import pprint\n",
    "pprint(tables.encoder_classes)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'BranchformerEncoder': <class 'funasr.models.branchformer.encoder.BranchformerEncoder'>,\n",
      " 'ChunkConformerEncoder': <class 'funasr.models.conformer.encoder.ConformerChunkEncoder'>,\n",
      " 'ConformerEncoder': <class 'funasr.models.conformer.encoder.ConformerEncoder'>,\n",
      " 'ConvBiasPredictor': <class 'funasr.models.lcbnet.encoder.ConvPredictor'>,\n",
      " 'DFSMN': <class 'funasr.models.fsmn_vad_streaming.encoder.DFSMN'>,\n",
      " 'EBranchformerEncoder': <class 'funasr.models.e_branchformer.encoder.EBranchformerEncoder'>,\n",
      " 'FSMN': <class 'funasr.models.fsmn_vad_streaming.encoder.FSMN'>,\n",
      " 'FSMNExport': <class 'funasr.models.fsmn_vad_streaming.encoder.FSMNExport'>,\n",
      " 'FusionSANEncoder': <class 'funasr.models.lcbnet.encoder.SelfSrcAttention'>,\n",
      " 'QwenAudioEncoder': <class 'funasr.models.qwen_audio.audio.QwenAudioEncoder'>,\n",
      " 'RWKVEncoder': <class 'funasr.models.rwkv_bat.rwkv_encoder.RWKVEncoder'>,\n",
      " 'SANMEncoder': <class 'funasr.models.sanm.encoder.SANMEncoder'>,\n",
      " 'SANMEncoderChunkOpt': <class 'funasr.models.scama.encoder.SANMEncoderChunkOpt'>,\n",
      " 'SANMEncoderChunkOptExport': <class 'funasr.models.sanm.encoder.SANMEncoderExport'>,\n",
      " 'SANMEncoderExport': <class 'funasr.models.sanm.encoder.SANMEncoderExport'>,\n",
      " 'SANMVadEncoder': <class 'funasr.models.ct_transformer_streaming.encoder.SANMVadEncoder'>,\n",
      " 'SANMVadEncoderExport': <class 'funasr.models.ct_transformer_streaming.encoder.SANMVadEncoderExport'>,\n",
      " 'SenseVoiceEncoderSmall': <class 'funasr.models.sense_voice.model.SenseVoiceEncoderSmall'>,\n",
      " 'TransformerEncoder': <class 'funasr.models.transformer.encoder.TransformerEncoder'>,\n",
      " 'TransformerTextEncoder': <class 'funasr.models.lcbnet.encoder.TransformerTextEncoder'>}\n"
     ]
    }
   ],
   "execution_count": 9
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-16T02:17:05.967615Z",
     "start_time": "2024-07-16T02:17:05.909015Z"
    }
   },
   "cell_type": "code",
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ],
   "outputs": [],
   "execution_count": 15
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-17T09:19:16.461368Z",
     "start_time": "2024-07-17T09:19:10.668026Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from funasr import AutoModel\n",
    "# paraformer-zh is a multi-functional asr model\n",
    "# use vad, punc, spk or not as you need\n",
    "auto_model = AutoModel(\n",
    "    # model='paraformer-zh',\n",
    "    model='iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch',\n",
    "    # model_conf='/home/perry/.cache/modelscope/hub/iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/configuration.json'\n",
    "    # vad_model=\"fsmn-vad\", vad_model_revision=\"v2.0.4\",\n",
    "    # punc_model=\"ct-punc-c\", punc_model_revision=\"v2.0.4\",\n",
    "    # spk_model=\"cam++\", spk_model_revision=\"v2.0.2\",\n",
    ")\n",
    "model, kwargs = auto_model.model, auto_model.kwargs\n",
    "kwargs['frontend'].fs = 22050\n",
    "\n",
    "asr_dir = outputs_dir / 'aishell3'\n",
    "os.makedirs(asr_dir, exist_ok=True)\n",
    "asr_dir = asr_dir / 'paraformer-large-zh'\n",
    "os.makedirs(asr_dir, exist_ok=True)"
   ],
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-07-17 17:19:11,350 - modelscope - WARNING - Using the master branch is fragile, please use it with caution!\n",
      "2024-07-17 17:19:11,350 - modelscope - INFO - Use user-specified model revision: master\n"
     ]
    }
   ],
   "execution_count": 84
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-15T23:35:57.507241Z",
     "start_time": "2024-07-15T23:35:41.196847Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from funasr import AutoModel\n",
    "# paraformer-zh is a multi-functional asr model\n",
    "# use vad, punc, spk or not as you need\n",
    "auto_model = AutoModel(\n",
    "    model='iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch',\n",
    "    # model='iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch', hub='ms',\n",
    "    # vad_model=\"fsmn-vad\", vad_model_revision=\"v2.0.4\",\n",
    "    # punc_model=\"ct-punc-c\", punc_model_revision=\"v2.0.4\",\n",
    "    # spk_model=\"cam++\", spk_model_revision=\"v2.0.2\",\n",
    ")\n",
    "model, kwargs = auto_model.model, auto_model.kwargs\n",
    "kwargs['frontend'].fs = 22050\n",
    "\n",
    "asr_dir = outputs_dir / 'aishell3'\n",
    "os.makedirs(asr_dir, exist_ok=True)\n",
    "asr_dir = asr_dir / 'paraformer-zh'\n",
    "os.makedirs(asr_dir, exist_ok=True)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are using the latest version of funasr-1.1.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-07-16 07:35:53,738 - modelscope - WARNING - Using the master branch is fragile, please use it with caution!\n",
      "2024-07-16 07:35:53,739 - modelscope - INFO - Use user-specified model revision: master\n"
     ]
    }
   ],
   "execution_count": 11
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-16T02:34:17.567772Z",
     "start_time": "2024-07-16T02:34:17.560438Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import numpy as np\n",
    "tokens = np.array(kwargs['token_list'])\n",
    "''.join(tokens[kwargs['suppress_idx']])"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"amcteiopsdfnlvhbrgwkuyjxqz''''ＡＴＭ\""
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 11
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-18T07:37:08.176015Z",
     "start_time": "2024-07-18T07:37:08.172453Z"
    }
   },
   "cell_type": "code",
   "source": [
    "gt_dir = data_dir / 'test/wav'\n",
    "# gt_paths = sorted(gt_dir.glob('*/*.wav'))\n",
    "gt_asr_path = asr_dir / 'gt_result.txt'\n",
    "\n",
    "# jets_paths = sorted(jets_dir.glob('*.wav'))\n",
    "jets_asr_path = asr_dir / 'jets_result.txt'\n",
    "\n",
    "# nopitch_paths = sorted(nopitch_dir.glob('*.wav'))\n",
    "nopitch_asr_path = asr_dir / 'nopitch_result.txt'\n",
    "\n",
    "# baseline_paths = sorted(nopitch_dir.glob('*.wav'))\n",
    "baseline_asr_path = asr_dir / 'baseline_result.txt'\n",
    "\n",
    "nodur_asr_path = asr_dir / 'nodur_result.txt'"
   ],
   "outputs": [],
   "execution_count": 111
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-17T09:19:16.507732Z",
     "start_time": "2024-07-17T09:19:16.466166Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import regex as re\n",
    "suppress_idx = []\n",
    "for i, token in enumerate(kwargs['tokenizer'].token_list):\n",
    "    if re.search(r'\\P{Han}', token) and not token.startswith('<'):\n",
    "        suppress_idx.append(i)\n",
    "kwargs['suppress_idx'] = suppress_idx"
   ],
   "outputs": [],
   "execution_count": 86
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-16T12:15:34.554188Z",
     "start_time": "2024-07-16T12:15:34.456680Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# Try running ASR on the first transcript\n",
    "model.eval()\n",
    "kwargs['fs'] = 22050\n",
    "mandarin_dir = jets_dir.parent / 'mandarin'\n",
    "result, _ = model.inference(data_in=str(jets_dir.parent / 'mandarin' / '四十四岁的徐女士报警称.wav'), key=['SSB06930030.wav'], **kwargs)\n",
    "print(result)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'key': 'SSB06930030.wav', 'text': '说声说说二词喂对谁去女女说吧谁你唱'}]\n"
     ]
    }
   ],
   "execution_count": 51
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-16T01:33:29.785756Z",
     "start_time": "2024-07-16T01:33:29.735141Z"
    }
   },
   "cell_type": "code",
   "source": "del kwargs['suppress_idx']",
   "outputs": [],
   "execution_count": 38
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-15T18:23:55.683003Z",
     "start_time": "2024-07-15T18:23:55.680609Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import logging\n",
    "logging.basicConfig(level=logging.WARNING)"
   ],
   "outputs": [],
   "execution_count": 45
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-17T09:19:16.512705Z",
     "start_time": "2024-07-17T09:19:16.508472Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from tqdm import tqdm\n",
    "def run_asr(filenames, audio_dir, asr_result_path, is_two_level=False, audio_fs=22050):\n",
    "    model.eval()\n",
    "    kwargs['fs'] = audio_fs\n",
    "    audio_dir = Path(audio_dir)\n",
    "    with open(asr_result_path, 'w') as f:\n",
    "        for filename in tqdm(filenames):\n",
    "            key = [filename]\n",
    "            if is_two_level:\n",
    "                data_in = str(audio_dir / filename[:7] / filename)\n",
    "            else:\n",
    "                data_in = str(audio_dir / filename)\n",
    "            try:\n",
    "                result, _ = model.inference(data_in=data_in, key=key, **kwargs)\n",
    "                text = result[0]['text']\n",
    "            except ValueError:\n",
    "                print('Error during inference for:', filename)\n",
    "                text = ''\n",
    "            f.write(f'{filename}|{text}\\n')"
   ],
   "outputs": [],
   "execution_count": 87
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-16T18:23:49.279285Z",
     "start_time": "2024-07-16T18:23:49.272683Z"
    }
   },
   "cell_type": "code",
   "source": "gt_asr_path",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('/home/perry/PycharmProjects/present/prosody/funasr/outputs/aishell3/aishell2-vocab5212/gt_result.txt')"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 65
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-17T09:52:31.306557Z",
     "start_time": "2024-07-17T09:19:16.513693Z"
    }
   },
   "cell_type": "code",
   "source": "run_asr(transcripts.keys(), gt_dir, gt_asr_path, is_two_level=True, audio_fs=44100)",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 24773/24773 [33:14<00:00, 12.42it/s]\n"
     ]
    }
   ],
   "execution_count": 88
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-17T11:37:45.767491Z",
     "start_time": "2024-07-17T09:52:31.307432Z"
    }
   },
   "cell_type": "code",
   "source": [
    "run_asr(transcripts.keys(), nopitch_dir, nopitch_asr_path)\n",
    "run_asr(transcripts.keys(), baseline_dir, baseline_asr_path)\n",
    "nodur_dir = prosody_outdir / 'tts_train_jets_raw_phn_tacotron_g2p_en_no_space/aishell3_nopitch_nodur'\n",
    "run_asr(transcripts.keys(), nodur_dir, nodur_asr_path)"
   ],
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 24773/24773 [31:23<00:00, 13.15it/s]\n",
      "100%|██████████| 24773/24773 [36:31<00:00, 11.30it/s]\n",
      "100%|██████████| 24773/24773 [37:19<00:00, 11.06it/s]\n"
     ]
    }
   ],
   "execution_count": 89
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": ""
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-16T16:23:50.173715Z",
     "start_time": "2024-07-16T15:57:15.200264Z"
    }
   },
   "cell_type": "code",
   "source": "run_asr(transcripts.keys(), jets_dir, jets_asr_path)",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▌    | 13717/24773 [14:52<09:39, 19.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error during inference for: SSB00090221.wav\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 14281/24773 [15:20<08:58, 19.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error during inference for: SSB03950012.wav\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 24773/24773 [26:34<00:00, 15.53it/s]\n"
     ]
    }
   ],
   "execution_count": 61
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-17T06:44:07.595473Z",
     "start_time": "2024-07-17T06:16:59.284630Z"
    }
   },
   "cell_type": "code",
   "source": "run_asr(transcripts.keys(), nopitch_dir, nopitch_asr_path)",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▌    | 13716/24773 [15:15<09:31, 19.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error during inference for: SSB00090179.wav\n",
      "Error during inference for: SSB00090221.wav\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 14281/24773 [15:44<08:46, 19.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error during inference for: SSB03950012.wav\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 24773/24773 [27:08<00:00, 15.21it/s]\n"
     ]
    }
   ],
   "execution_count": 71
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-17T07:16:43.769450Z",
     "start_time": "2024-07-17T06:44:07.596991Z"
    }
   },
   "cell_type": "code",
   "source": [
    "baseline_dir = prosody_outdir / 'zm-text-tts' / 'aishell3'\n",
    "run_asr(transcripts.keys(), baseline_dir, baseline_asr_path)"
   ],
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|████▎     | 10716/24773 [14:26<18:13, 12.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error during inference for: SSB13820236.wav\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|█████▏    | 12801/24773 [17:16<14:09, 14.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error during inference for: SSB18100419.wav\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▌    | 13715/24773 [18:33<11:15, 16.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error during inference for: SSB00090179.wav\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|█████▉    | 14754/24773 [19:35<10:12, 16.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error during inference for: SSB06860272.wav\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 21787/24773 [28:51<04:08, 11.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error during inference for: SSB14370165.wav\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 24773/24773 [32:36<00:00, 12.66it/s]\n"
     ]
    }
   ],
   "execution_count": 72
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-17T08:44:16.940713Z",
     "start_time": "2024-07-17T08:44:16.937688Z"
    }
   },
   "cell_type": "code",
   "source": "nodur_dir",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('/home/perry/PycharmProjects/present/prosody/outputs/tts_train_jets_raw_phn_tacotron_g2p_en_no_space/aishell3_nopitchnodur')"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 81
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-17T09:19:10.666920Z",
     "start_time": "2024-07-17T08:44:57.383387Z"
    }
   },
   "cell_type": "code",
   "source": "run_asr(transcripts.keys(), nodur_dir, nodur_asr_path)",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 24773/24773 [34:13<00:00, 12.07it/s]\n"
     ]
    }
   ],
   "execution_count": 83
  },
  {
   "cell_type": "code",
   "source": [
    "import re\n",
    "import jiwer\n",
    "def eval_cer(transcripts, asr_result_path, cer_path):\n",
    "    with open(cer_path, 'w') as cer_file:\n",
    "        cer_file.write('wav_file,hanzi_len,hanzi_cer,pinyin_len,pinyin_cer\\n')\n",
    "        with open(asr_result_path) as f:\n",
    "            for line in f:\n",
    "                wav_file, asr_hans = line.strip('\\n').split('|', maxsplit=1)\n",
    "                asr_pinyins = ''.join(hans_to_pinyins(asr_hans)) \n",
    "                # eng_chars = sum([word.isascii() for word in asr_output])\n",
    "                hans, pinyins = transcripts[wav_file]\n",
    "                pinyins = re.sub(r'\\s', '', pinyins)\n",
    "                hanzi_len = len(hans)\n",
    "                pinyin_len = len(pinyins)\n",
    "                hanzi_cer = jiwer.cer(reference=hans, hypothesis=asr_hans)\n",
    "                pinyin_cer = jiwer.cer(reference=pinyins, hypothesis=asr_pinyins)\n",
    "                cer_file.write(f'{wav_file},{hanzi_len},{hanzi_cer},{pinyin_len},{pinyin_cer}\\n')\n",
    "                "
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-07-17T11:56:07.331554Z",
     "start_time": "2024-07-17T11:56:07.326944Z"
    }
   },
   "outputs": [],
   "execution_count": 90
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-16T02:20:33.983447Z",
     "start_time": "2024-07-16T02:20:33.910437Z"
    }
   },
   "cell_type": "code",
   "source": "hans_to_pinyins(hans)",
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mIndexError\u001B[0m                                Traceback (most recent call last)",
      "Cell \u001B[0;32mIn[25], line 1\u001B[0m\n\u001B[0;32m----> 1\u001B[0m \u001B[43mhans_to_pinyins\u001B[49m\u001B[43m(\u001B[49m\u001B[43mhans\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/PycharmProjects/present/prosody/funasr/prosody/pinyin.py:237\u001B[0m, in \u001B[0;36mhans_to_pinyins\u001B[0;34m(hans)\u001B[0m\n\u001B[1;32m    235\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m i \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m0\u001B[39m \u001B[38;5;129;01mor\u001B[39;00m hans[i \u001B[38;5;241m-\u001B[39m \u001B[38;5;241m1\u001B[39m] \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;124m'\u001B[39m\u001B[38;5;124m〇零一二三四五六七八九十\u001B[39m\u001B[38;5;124m'\u001B[39m:\n\u001B[1;32m    236\u001B[0m     yi_pinyin \u001B[38;5;241m=\u001B[39m with_sandhi[i]\n\u001B[0;32m--> 237\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m \u001B[43mwith_sandhi\u001B[49m\u001B[43m[\u001B[49m\u001B[43mi\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m+\u001B[39;49m\u001B[43m \u001B[49m\u001B[38;5;241;43m1\u001B[39;49m\u001B[43m]\u001B[49m\u001B[38;5;241m.\u001B[39mendswith(\u001B[38;5;124m'\u001B[39m\u001B[38;5;124m4\u001B[39m\u001B[38;5;124m'\u001B[39m):\n\u001B[1;32m    238\u001B[0m         with_sandhi[i] \u001B[38;5;241m=\u001B[39m yi_pinyin[:\u001B[38;5;241m-\u001B[39m\u001B[38;5;241m1\u001B[39m] \u001B[38;5;241m+\u001B[39m \u001B[38;5;124m'\u001B[39m\u001B[38;5;124m2\u001B[39m\u001B[38;5;124m'\u001B[39m\n\u001B[1;32m    239\u001B[0m     \u001B[38;5;28;01melse\u001B[39;00m:\n",
      "\u001B[0;31mIndexError\u001B[0m: list index out of range"
     ]
    }
   ],
   "execution_count": 25
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-18T07:37:00.943071Z",
     "start_time": "2024-07-18T07:37:00.940778Z"
    }
   },
   "cell_type": "code",
   "source": "asr_dir = asr_dir.parent / 'paraformer-large-zh'",
   "outputs": [],
   "execution_count": 110
  },
  {
   "cell_type": "code",
   "source": [
    "jets_cer_path = asr_dir / 'jets_cer.csv'\n",
    "eval_cer(transcripts=transcripts, asr_result_path=jets_asr_path, cer_path=jets_cer_path)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-07-16T16:32:22.398858Z",
     "start_time": "2024-07-16T16:32:16.622649Z"
    }
   },
   "outputs": [],
   "execution_count": 63
  },
  {
   "cell_type": "code",
   "source": [
    "nopitch_cer_path = asr_dir / 'nopitch_cer.csv'\n",
    "eval_cer(transcripts=transcripts, asr_result_path=nopitch_asr_path, cer_path=nopitch_cer_path)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-07-18T07:37:19.705466Z",
     "start_time": "2024-07-18T07:37:13.286225Z"
    }
   },
   "outputs": [],
   "execution_count": 112
  },
  {
   "cell_type": "code",
   "source": [
    "baseline_cer_path = asr_dir / 'baseline_cer.csv'\n",
    "eval_cer(transcripts=transcripts, asr_result_path=baseline_asr_path, cer_path=baseline_cer_path)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-07-18T07:37:24.425792Z",
     "start_time": "2024-07-18T07:37:19.706857Z"
    }
   },
   "outputs": [],
   "execution_count": 113
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-18T07:35:40.163862Z",
     "start_time": "2024-07-18T07:35:40.159636Z"
    }
   },
   "cell_type": "code",
   "source": "",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('/home/perry/PycharmProjects/present/prosody/funasr/outputs/aishell3/aishell2-vocab5212/gt_cer.csv')"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 103
  },
  {
   "cell_type": "code",
   "source": [
    "gt_cer_path = asr_dir / 'gt_cer.csv'\n",
    "eval_cer(transcripts=transcripts, asr_result_path=gt_asr_path, cer_path=gt_cer_path)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-07-18T07:37:29.112731Z",
     "start_time": "2024-07-18T07:37:24.426803Z"
    }
   },
   "outputs": [],
   "execution_count": 114
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-18T07:37:38.036199Z",
     "start_time": "2024-07-18T07:37:29.114007Z"
    }
   },
   "cell_type": "code",
   "source": [
    "nodur_cer_path = asr_dir / 'nodur_cer.csv'\n",
    "eval_cer(transcripts=transcripts, asr_result_path=nodur_asr_path, cer_path=nodur_cer_path)"
   ],
   "outputs": [],
   "execution_count": 115
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}