|
| 1 | +import hashlib |
| 2 | +import json |
| 3 | +import os |
| 4 | +import shutil |
| 5 | +from pathlib import Path |
| 6 | +from stat import S_ISREG, ST_MTIME, ST_MODE, ST_SIZE |
| 7 | + |
| 8 | +from ovos_utils.file_utils import get_cache_directory |
| 9 | +from ovos_utils.log import LOG |
| 10 | + |
| 11 | + |
| 12 | +def hash_sentence(sentence: str): |
| 13 | + """Convert the sentence into a hash value used for the file name |
| 14 | +
|
| 15 | + Args: |
| 16 | + sentence: The sentence to be cached |
| 17 | + """ |
| 18 | + encoded_sentence = sentence.encode("utf-8", "ignore") |
| 19 | + sentence_hash = hashlib.md5(encoded_sentence).hexdigest() |
| 20 | + return sentence_hash |
| 21 | + |
| 22 | + |
| 23 | +def hash_from_path(path: Path) -> str: |
| 24 | + """Returns hash from a given path. |
| 25 | +
|
| 26 | + Simply removes extension and folder structure leaving the hash. |
| 27 | +
|
| 28 | + NOTE: this does not do any hashing at all and naming is misleading |
| 29 | + however we keep the method around for backwards compat imports |
| 30 | + this is exclusively for usage with cached TTS files |
| 31 | +
|
| 32 | + Args: |
| 33 | + path: path to get hash from |
| 34 | +
|
| 35 | + Returns: |
| 36 | + Hash reference for file. |
| 37 | + """ |
| 38 | + # NOTE: this does not do any hashing at all and naming is misleading |
| 39 | + # however we keep the method around for backwards compat imports |
| 40 | + # this is assumed to be used only to load cached TTS which is already named with an hash |
| 41 | + return path.with_suffix('').name |
| 42 | + |
| 43 | + |
| 44 | +def mb_to_bytes(size): |
| 45 | + """Takes a size in MB and returns the number of bytes. |
| 46 | +
|
| 47 | + Args: |
| 48 | + size(int/float): size in Mega Bytes |
| 49 | +
|
| 50 | + Returns: |
| 51 | + (int/float) size in bytes |
| 52 | + """ |
| 53 | + return size * 1024 * 1024 |
| 54 | + |
| 55 | + |
| 56 | +def _get_cache_entries(directory): |
| 57 | + """Get information tuple for all regular files in directory. |
| 58 | +
|
| 59 | + Args: |
| 60 | + directory (str): path to directory to check |
| 61 | +
|
| 62 | + Returns: |
| 63 | + (tuple) (modification time, size, filepath) |
| 64 | + """ |
| 65 | + entries = (os.path.join(directory, fn) for fn in os.listdir(directory)) |
| 66 | + entries = ((os.stat(path), path) for path in entries) |
| 67 | + |
| 68 | + # leave only regular files, insert modification date |
| 69 | + return ((stat[ST_MTIME], stat[ST_SIZE], path) |
| 70 | + for stat, path in entries if S_ISREG(stat[ST_MODE])) |
| 71 | + |
| 72 | + |
| 73 | +def _delete_oldest(entries, bytes_needed): |
| 74 | + """Delete files with oldest modification date until space is freed. |
| 75 | +
|
| 76 | + Args: |
| 77 | + entries (tuple): file + file stats tuple |
| 78 | + bytes_needed (int): disk space that needs to be freed |
| 79 | +
|
| 80 | + Returns: |
| 81 | + (list) all removed paths |
| 82 | + """ |
| 83 | + deleted_files = [] |
| 84 | + space_freed = 0 |
| 85 | + for moddate, fsize, path in sorted(entries): |
| 86 | + try: |
| 87 | + os.remove(path) |
| 88 | + space_freed += fsize |
| 89 | + deleted_files.append(path) |
| 90 | + except Exception: |
| 91 | + pass |
| 92 | + |
| 93 | + if space_freed > bytes_needed: |
| 94 | + break # deleted enough! |
| 95 | + |
| 96 | + return deleted_files |
| 97 | + |
| 98 | + |
| 99 | +def curate_cache(directory, min_free_percent=5.0, min_free_disk=50): |
| 100 | + """Clear out the directory if needed. |
| 101 | +
|
| 102 | + The curation will only occur if both the precentage and actual disk space |
| 103 | + is below the limit. This assumes all the files in the directory can be |
| 104 | + deleted as freely. |
| 105 | +
|
| 106 | + Args: |
| 107 | + directory (str): directory path that holds cached files |
| 108 | + min_free_percent (float): percentage (0.0-100.0) of drive to keep free, |
| 109 | + default is 5% if not specified. |
| 110 | + min_free_disk (float): minimum allowed disk space in MB, default |
| 111 | + value is 50 MB if not specified. |
| 112 | + """ |
| 113 | + # Simpleminded implementation -- keep a certain percentage of the |
| 114 | + # disk available. |
| 115 | + # TODO: Would be easy to add more options, like whitelisted files, etc. |
| 116 | + deleted_files = [] |
| 117 | + |
| 118 | + # Get the disk usage statistics bout the given path |
| 119 | + space = shutil.disk_usage(directory) |
| 120 | + |
| 121 | + percent_free = space.free * 100 / space.total |
| 122 | + |
| 123 | + min_free_disk = mb_to_bytes(min_free_disk) |
| 124 | + |
| 125 | + if percent_free < min_free_percent and space.free < min_free_disk: |
| 126 | + LOG.info('Low diskspace detected, cleaning cache') |
| 127 | + # calculate how many bytes we need to delete |
| 128 | + bytes_needed = (min_free_percent - percent_free) / 100.0 * space.total |
| 129 | + bytes_needed = int(bytes_needed + 1.0) |
| 130 | + |
| 131 | + # get all entries in the directory w/ stats |
| 132 | + entries = _get_cache_entries(directory) |
| 133 | + # delete as many as needed starting with the oldest |
| 134 | + deleted_files = _delete_oldest(entries, bytes_needed) |
| 135 | + |
| 136 | + return deleted_files |
| 137 | + |
| 138 | + |
| 139 | +class AudioFile: |
| 140 | + def __init__(self, cache_dir: Path, sentence_hash: str, file_type: str): |
| 141 | + self.name = f"{sentence_hash}.{file_type}" |
| 142 | + self.path = cache_dir.joinpath(self.name) |
| 143 | + |
| 144 | + def save(self, audio: bytes): |
| 145 | + """Write a TTS cache file containing the audio to be spoken. |
| 146 | + Args: |
| 147 | + audio: TTS inference of a sentence |
| 148 | + """ |
| 149 | + try: |
| 150 | + with open(self.path, "wb") as audio_file: |
| 151 | + audio_file.write(audio) |
| 152 | + except Exception: |
| 153 | + LOG.exception("Failed to write {} to cache".format(self.name)) |
| 154 | + |
| 155 | + def exists(self): |
| 156 | + return self.path.exists() |
| 157 | + |
| 158 | + |
| 159 | +class PhonemeFile: |
| 160 | + def __init__(self, cache_dir: Path, sentence_hash: str): |
| 161 | + self.name = f"{sentence_hash}.pho" |
| 162 | + self.path = cache_dir.joinpath(self.name) |
| 163 | + |
| 164 | + def load(self): |
| 165 | + """Load phonemes from cache file.""" |
| 166 | + phonemes = None |
| 167 | + if self.path.exists(): |
| 168 | + try: |
| 169 | + with open(self.path) as phoneme_file: |
| 170 | + phonemes = phoneme_file.read().strip() |
| 171 | + except Exception: |
| 172 | + LOG.exception("Failed to read phoneme from cache") |
| 173 | + |
| 174 | + return json.loads(phonemes) |
| 175 | + |
| 176 | + def save(self, phonemes): |
| 177 | + """Write a TTS cache file containing the phoneme to be displayed. |
| 178 | + Args: |
| 179 | + phonemes: instructions for how to make the mouth on a device move |
| 180 | + """ |
| 181 | + try: |
| 182 | + rec = json.dumps(phonemes) |
| 183 | + with open(self.path, "w") as phoneme_file: |
| 184 | + phoneme_file.write(rec) |
| 185 | + except Exception: |
| 186 | + LOG.error(f"Failed to write {self.name} to cache") |
| 187 | + |
| 188 | + def exists(self): |
| 189 | + return self.path.exists() |
| 190 | + |
| 191 | + |
| 192 | +class TextToSpeechCache: |
| 193 | + """Class for all persistent and temporary caching operations.""" |
| 194 | + |
| 195 | + def __init__(self, tts_config, tts_name, audio_file_type): |
| 196 | + self.config = tts_config |
| 197 | + self.tts_name = tts_name |
| 198 | + if "preloaded_cache" in self.config: |
| 199 | + self.persistent_cache_dir = Path(self.config["preloaded_cache"]) |
| 200 | + os.makedirs(str(self.persistent_cache_dir), exist_ok=True) |
| 201 | + else: |
| 202 | + self.persistent_cache_dir = None |
| 203 | + self.temporary_cache_dir = Path( |
| 204 | + get_cache_directory("tts/" + tts_name) |
| 205 | + ) |
| 206 | + os.makedirs(str(self.temporary_cache_dir), exist_ok=True) |
| 207 | + self.audio_file_type = audio_file_type |
| 208 | + self.cached_sentences = {} |
| 209 | + # curate cache if disk usage is above min % |
| 210 | + self.min_free_percent = self.config.get("min_free_percent", 75) |
| 211 | + |
| 212 | + def __contains__(self, sha): |
| 213 | + """The cache contains a SHA if it knows of it and it exists on disk.""" |
| 214 | + if sha not in self.cached_sentences: |
| 215 | + return False # Doesn't know of it |
| 216 | + else: |
| 217 | + # Audio file must exist, phonemes are optional. |
| 218 | + audio, phonemes = self.cached_sentences[sha] |
| 219 | + return (audio.exists() and |
| 220 | + (phonemes is None or phonemes.exists())) |
| 221 | + |
| 222 | + def load_persistent_cache(self): |
| 223 | + """Load the contents of dialog files to the persistent cache directory. |
| 224 | + Parse the dialog files in the resource directory into sentences. Then |
| 225 | + add the audio for each sentence to the cache directory. |
| 226 | + NOTE: There may be files pre-loaded in the persistent cache directory |
| 227 | + prior to run time, such as pre-recorded audio files. This will add |
| 228 | + files that do not already exist. |
| 229 | + ANOTHER NOTE: Mimic2 is the only TTS engine that supports |
| 230 | + downloading missing files. This logic will need to change if another |
| 231 | + TTS engine implements it. |
| 232 | + """ |
| 233 | + if self.persistent_cache_dir is not None: |
| 234 | + LOG.info("Adding dialog resources to persistent TTS cache...") |
| 235 | + self._load_existing_audio_files() |
| 236 | + self._load_existing_phoneme_files() |
| 237 | + LOG.info("Persistent TTS cache files added successfully.") |
| 238 | + |
| 239 | + def _load_existing_audio_files(self): |
| 240 | + """Find the TTS audio files already in the persistent cache.""" |
| 241 | + glob_pattern = "*." + self.audio_file_type |
| 242 | + for file_path in self.persistent_cache_dir.glob(glob_pattern): |
| 243 | + sentence_hash = file_path.name.split(".")[0] |
| 244 | + audio_file = AudioFile( |
| 245 | + self.persistent_cache_dir, sentence_hash, self.audio_file_type |
| 246 | + ) |
| 247 | + self.cached_sentences[sentence_hash] = audio_file, None |
| 248 | + |
| 249 | + def _load_existing_phoneme_files(self): |
| 250 | + """Find the TTS phoneme files already in the persistent cache. |
| 251 | + A phoneme file is no good without an audio file to pair it with. If |
| 252 | + no audio file matches, do not load the phoneme. |
| 253 | + """ |
| 254 | + for file_path in self.persistent_cache_dir.glob("*.pho"): |
| 255 | + sentence_hash = file_path.name.split(".")[0] |
| 256 | + cached_sentence = self.cached_sentences.get(sentence_hash) |
| 257 | + if cached_sentence is not None: |
| 258 | + audio_file = cached_sentence[0] |
| 259 | + phoneme_file = PhonemeFile( |
| 260 | + self.persistent_cache_dir, sentence_hash |
| 261 | + ) |
| 262 | + self.cached_sentences[sentence_hash] = audio_file, phoneme_file |
| 263 | + |
| 264 | + def clear(self): |
| 265 | + """Remove all files from the temporary cache.""" |
| 266 | + for cache_file_path in self.temporary_cache_dir.iterdir(): |
| 267 | + if cache_file_path.is_dir(): |
| 268 | + for sub_path in cache_file_path.iterdir(): |
| 269 | + if sub_path.is_file(): |
| 270 | + sub_path.unlink() |
| 271 | + elif cache_file_path.is_file(): |
| 272 | + cache_file_path.unlink() |
| 273 | + |
| 274 | + def curate(self): |
| 275 | + """Remove cache data if disk space is running low.""" |
| 276 | + files_removed = curate_cache(str(self.temporary_cache_dir), |
| 277 | + min_free_percent=self.min_free_percent) |
| 278 | + hashes = set([hash_from_path(Path(path)) for path in files_removed]) |
| 279 | + for sentence_hash in hashes: |
| 280 | + if sentence_hash in self.cached_sentences: |
| 281 | + self.cached_sentences.pop(sentence_hash) |
| 282 | + |
| 283 | + def define_audio_file(self, sentence_hash: str) -> AudioFile: |
| 284 | + """Build an instance of an object representing an audio file.""" |
| 285 | + audio_file = AudioFile( |
| 286 | + self.temporary_cache_dir, sentence_hash, self.audio_file_type |
| 287 | + ) |
| 288 | + return audio_file |
| 289 | + |
| 290 | + def define_phoneme_file(self, sentence_hash: str) -> PhonemeFile: |
| 291 | + """Build an instance of an object representing an phoneme file.""" |
| 292 | + phoneme_file = PhonemeFile(self.temporary_cache_dir, sentence_hash) |
| 293 | + return phoneme_file |
0 commit comments