Update Proficiency 0.5.21

and combine Wiktionary and Kindle gloss language config value

config.py

@@ -53,8 +53,7 @@
 prefs.defaults["use_all_formats"] = False
 prefs.defaults["minimal_x_ray_count"] = 1
 prefs.defaults["choose_format_manually"] = True
-prefs.defaults["wiktionary_gloss_lang"] = "en"
-prefs.defaults["kindle_gloss_lang"] = "en"
+prefs.defaults["gloss_lang"] = "en"
 prefs.defaults["use_wiktionary_for_kindle"] = False
 prefs.defaults["remove_link_styles"] = False
 prefs.defaults["python_path"] = ""
@@ -220,9 +219,7 @@ def open_choose_lemma_lang_dialog(self, is_kindle: bool = True) -> None:
  if choose_lang_dlg.exec():
  lemma_lang = choose_lang_dlg.lemma_lang_box.currentData()
  gloss_lang = choose_lang_dlg.gloss_lang_box.currentData()
- prefs["kindle_gloss_lang" if is_kindle else "wiktionary_gloss_lang"] = (
- gloss_lang
- )
+ prefs["gloss_lang"] = gloss_lang
  if is_kindle and lemma_lang == "en" and gloss_lang in ["en", "zh", "zh_cn"]:
  prefs["use_wiktionary_for_kindle"] = (
  choose_lang_dlg.use_wiktionary_box.isChecked()
@@ -436,10 +433,6 @@ class ChooseLemmaLangDialog(QDialog):
  def __init__(self, parent: QObject, is_kindle: bool):
  super().__init__(parent)
  self.setWindowTitle(_("Choose language"))
- self.prefer_gloss_code = prefs[
- "kindle_gloss_lang" if is_kindle else "wiktionary_gloss_lang"
- ]
-
  form_layout = QFormLayout()
  form_layout.setFieldGrowthPolicy(
  QFormLayout.FieldGrowthPolicy.ExpandingFieldsGrow
@@ -448,14 +441,12 @@ def __init__(self, parent: QObject, is_kindle: bool):
  self.lemma_lang_box = QComboBox()
  self.gloss_lang_box = QComboBox()
  language_dict = load_languages_data(get_plugin_path())
- selected_gloss_code = prefs[
- "kindle_gloss_lang" if is_kindle else "wiktionary_gloss_lang"
- ]
+ selected_gloss_code = prefs["gloss_lang"]
  self.gloss_lang_box.currentIndexChanged.connect(
  partial(self.gloss_lang_changed, language_dict)
  )
  for gloss_lang, lang_value in language_dict.items():
- if len(lang_value.get("lemma_languages", [])) == 0:
+ if lang_value.get("gloss_source", []) == "":
  continue
  gloss_lang_name = _(lang_value["name"])
  self.gloss_lang_box.addItem(gloss_lang_name, gloss_lang)
@@ -503,6 +494,8 @@ def gloss_lang_changed(self, lang_dict) -> None:
  gloss_lang = self.gloss_lang_box.currentData()
  self.lemma_lang_box.clear()
  lemma_langs = lang_dict[gloss_lang].get("lemma_languages", [])
+ if len(lemma_langs) == 0:
+ lemma_langs = lang_dict.keys()
  for index, lemma_lang in enumerate(lemma_langs):
  lemma_lang_name = _(lang_dict[lemma_lang]["name"])
  self.lemma_lang_box.addItem(lemma_lang_name, lemma_lang)

custom_lemmas.py

@@ -210,7 +210,7 @@ def check_empty_kindle_gloss(self) -> None:
  if not package_name:
  device_not_found_dialog(self)
  return
- custom_folder = custom_lemmas_folder(plugin_path, "en")
+ custom_folder = custom_lemmas_folder(plugin_path)
  if isinstance(package_name, str):
  copy_klld_from_android(package_name, custom_folder)
  else:

data/languages.json

@@ -24,34 +24,6 @@
  "639-2": "deu",
  "gloss_source": "kaikki",
  "has_trf": false,
- "lemma_languages": [
- "ca",
- "cs",
- "da",
- "de",
- "el",
- "en",
- "es",
- "fi",
- "fr",
- "he",
- "hr",
- "it",
- "ja",
- "ko",
- "lt",
- "mk",
- "nl",
- "no",
- "pl",
- "pt",
- "ro",
- "ru",
- "sl",
- "sv",
- "uk",
- "zh"
- ],
  "name": "German",
  "spacy": "de_core_news_"
  },
@@ -71,69 +43,13 @@
  "639-2": "eng",
  "gloss_source": "kaikki",
  "has_trf": true,
- "lemma_languages": [
- "ca",
- "cs",
- "da",
- "de",
- "el",
- "en",
- "es",
- "fi",
- "fr",
- "he",
- "hr",
- "it",
- "ja",
- "ko",
- "lt",
- "mk",
- "nl",
- "no",
- "pl",
- "pt",
- "ro",
- "ru",
- "sl",
- "sv",
- "uk",
- "zh"
- ],
  "name": "English",
  "spacy": "en_core_web_"
  },
  "es": {
  "639-2": "spa",
  "gloss_source": "kaikki",
  "has_trf": false,
- "lemma_languages": [
- "ca",
- "cs",
- "da",
- "de",
- "el",
- "en",
- "es",
- "fi",
- "fr",
- "he",
- "hr",
- "it",
- "ja",
- "ko",
- "lt",
- "mk",
- "nl",
- "no",
- "pl",
- "pt",
- "ro",
- "ru",
- "sl",
- "sv",
- "uk",
- "zh"
- ],
  "name": "Spanish",
  "spacy": "es_core_news_"
  },
@@ -151,34 +67,6 @@
  "639-2": "fra",
  "gloss_source": "kaikki",
  "has_trf": false,
- "lemma_languages": [
- "ca",
- "cs",
- "da",
- "de",
- "el",
- "en",
- "es",
- "fi",
- "fr",
- "he",
- "hr",
- "it",
- "ja",
- "ko",
- "lt",
- "mk",
- "nl",
- "no",
- "pl",
- "pt",
- "ro",
- "ru",
- "sl",
- "sv",
- "uk",
- "zh"
- ],
  "name": "French",
  "spacy": "fr_core_news_"
  },
@@ -215,11 +103,8 @@
  },
  "ja": {
  "639-2": "jpn",
- "gloss_source": "dbnary",
+ "gloss_source": "kaikki",
  "has_trf": true,
- "lemma_languages": [
- "ja"
- ],
  "name": "Japanese",
  "spacy": "ja_core_news_"
  },
@@ -257,7 +142,7 @@
  "name": "Dutch",
  "spacy": "nl_core_news_"
  },
- "no": {
+ "nb": {
  "639-2": "nob",
  "gloss_source": "dbnary",
  "has_trf": false,
@@ -269,11 +154,8 @@
  },
  "pl": {
  "639-2": "pol",
- "gloss_source": "dbnary",
+ "gloss_source": "kaikki",
  "has_trf": false,
- "lemma_languages": [
- "pl"
- ],
  "name": "Polish",
  "spacy": "pl_core_news_"
  },
@@ -298,34 +180,6 @@
  "639-2": "rus",
  "gloss_source": "kaikki",
  "has_trf": false,
- "lemma_languages": [
- "ca",
- "cs",
- "da",
- "de",
- "el",
- "en",
- "es",
- "fi",
- "fr",
- "he",
- "hr",
- "it",
- "ja",
- "ko",
- "lt",
- "mk",
- "nl",
- "no",
- "pl",
- "pt",
- "ro",
- "ru",
- "sl",
- "sv",
- "uk",
- "zh"
- ],
  "name": "Russian",
  "spacy": "ru_core_news_"
  },
@@ -357,34 +211,6 @@
  "639-2": "zho",
  "gloss_source": "kaikki",
  "has_trf": true,
- "lemma_languages": [
- "ca",
- "cs",
- "da",
- "de",
- "el",
- "en",
- "es",
- "fi",
- "fr",
- "he",
- "hr",
- "it",
- "ja",
- "ko",
- "lt",
- "mk",
- "nl",
- "no",
- "pl",
- "pt",
- "ro",
- "ru",
- "sl",
- "sv",
- "uk",
- "zh"
- ],
  "name": "Chinese",
  "spacy": "zh_core_web_"
  }

deps.py

@@ -1,6 +1,6 @@
-import bz2
 import platform
 import shutil
+import tarfile
 from pathlib import Path
 from typing import Any
 from urllib.request import urlopen
@@ -10,14 +10,12 @@
 from .utils import (
  PROFICIENCY_RELEASE_URL,
  Prefs,
+ custom_lemmas_folder,
  get_plugin_path,
  get_spacy_model_version,
- get_wiktionary_klld_path,
- kindle_db_path,
  load_plugin_json,
  mac_bin_path,
  run_subprocess,
- wiktionary_db_path,
 )
 
 PY_PATH = ""
@@ -146,7 +144,7 @@ def download_word_wise_file(
  log=None,
  notifications=None,
 ) -> None:
- gloss_lang = prefs["kindle_gloss_lang" if is_kindle else "wiktionary_gloss_lang"]
+ gloss_lang = prefs["gloss_lang"]
  if notifications:
  notifications.put(
  (
@@ -156,27 +154,14 @@ def download_word_wise_file(
  )
  )
  plugin_path = get_plugin_path()
- if is_kindle:
- db_path = kindle_db_path(plugin_path, lemma_lang, prefs)
- else:
- db_path = wiktionary_db_path(plugin_path, lemma_lang, gloss_lang)
-
- if not db_path.exists():
- bz2_filename = db_path.with_suffix(db_path.suffix + ".bz2").name
- download_and_extract(f"{PROFICIENCY_RELEASE_URL}/{bz2_filename}", db_path)
-
- if is_kindle:
- klld_path = get_wiktionary_klld_path(plugin_path, lemma_lang, gloss_lang)
- if not klld_path.exists():
- bz2_filename = klld_path.with_suffix(klld_path.suffix + ".bz2").name
- download_and_extract(f"{PROFICIENCY_RELEASE_URL}/{bz2_filename}", klld_path)
-
-
-def download_and_extract(url: str, extract_path: Path) -> None:
- extract_path.parent.mkdir(parents=True, exist_ok=True)
- download_path = extract_path.with_name(url.rsplit("/", 1)[-1])
+ bz2_filename = f"{lemma_lang}_{gloss_lang}.tar.bz2"
+ url = f"{PROFICIENCY_RELEASE_URL}/{bz2_filename}"
+ download_folder = custom_lemmas_folder(plugin_path)
+ if not download_folder.is_dir():
+ download_folder.mkdir()
+ download_path = download_folder / bz2_filename
  with urlopen(url) as r, open(download_path, "wb") as f:
  shutil.copyfileobj(r, f)
- with bz2.open(download_path) as bz2_f, extract_path.open("wb") as f:
- shutil.copyfileobj(bz2_f, f)
+ with tarfile.open(name=download_path, mode="r:bz2") as tar_f:
+ tar_f.extractall(download_folder)
  download_path.unlink()