diff --git a/.github/workflows/auto_translate.yml b/.github/workflows/auto_translate.yml new file mode 100644 index 0000000..409a93e --- /dev/null +++ b/.github/workflows/auto_translate.yml @@ -0,0 +1,29 @@ +name: Auto translate +on: + workflow_dispatch: + +jobs: + autotranslate: + env: + API_KEY: ${{secrets.DL_API_KEY}} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: dev + fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: Install Translate Tools + run: | + python -m pip install ovos-translate-plugin-deepl ovos-utils + - name: Auto Translate + run: | + python scripts/translate.py + - name: Commit to dev + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: autotranslate + branch: dev diff --git a/.github/workflows/build_tests.yml b/.github/workflows/build_tests.yml new file mode 100644 index 0000000..d6dbee7 --- /dev/null +++ b/.github/workflows/build_tests.yml @@ -0,0 +1,39 @@ +name: Run Build Tests +on: + push: + branches: + - master + pull_request: + branches: + - dev + workflow_dispatch: + +jobs: + py_build_tests: + uses: neongeckocom/.github/.github/workflows/python_build_tests.yml@master + with: + test_manifest: true + manifest_ignored: "test/**,scripts/**,CHANGELOG.md" + pip_audit: + strategy: + max-parallel: 2 + matrix: + python-version: ["3.10"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install skill + run: | + pip install . + - uses: pypa/gh-action-pip-audit@v1.0.7 + with: + # Ignore setuptools vulnerability we can't do much about + # Ignore numpy vulnerability affecting latest version for Py3.7 + ignore-vulns: | + GHSA-r9hx-vwmv-q579 + GHSA-fpfv-jqm9-f5jm + PYSEC-2022-43012 diff --git a/.github/workflows/dev2master.yml b/.github/workflows/dev2master.yml new file mode 100644 index 0000000..cc76fee --- /dev/null +++ b/.github/workflows/dev2master.yml @@ -0,0 +1,20 @@ +# This workflow will generate a distribution and upload it to PyPI + +name: Push dev -> master +on: + workflow_dispatch: + +jobs: + build_and_publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. + ref: dev + - name: Push dev -> master + uses: ad-m/github-push-action@master + + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + branch: master \ No newline at end of file diff --git a/.github/workflows/license_tests.yml b/.github/workflows/license_tests.yml new file mode 100644 index 0000000..46c73f4 --- /dev/null +++ b/.github/workflows/license_tests.yml @@ -0,0 +1,44 @@ +name: Run License Tests +on: + push: + branches: + - master + pull_request: + branches: + - dev + workflow_dispatch: + +jobs: + license_tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install Build Tools + run: | + python -m pip install build wheel + - name: Install System Dependencies + run: | + sudo apt-get update + sudo apt install python3-dev swig libssl-dev + - name: Install core repo + run: | + pip install . + - name: Get explicit and transitive dependencies + run: | + pip freeze > requirements-all.txt + - name: Check python + id: license_check_report + uses: pilosus/action-pip-license-checker@v0.5.0 + with: + requirements: 'requirements-all.txt' + fail: 'Copyleft,Other,Error' + fails-only: true + exclude: '^(tqdm|ovos-skill-).*' + exclude-license: '^(Mozilla).*$' + - name: Print report + if: ${{ always() }} + run: echo "${{ steps.license_check_report.outputs.report }}" \ No newline at end of file diff --git a/.github/workflows/propose_translation.yml b/.github/workflows/propose_translation.yml new file mode 100644 index 0000000..cb03a37 --- /dev/null +++ b/.github/workflows/propose_translation.yml @@ -0,0 +1,55 @@ +name: Propose Translation +on: + workflow_dispatch: + inputs: + language: + type: choice + description: Language to translate + options: + - de-de + - ca-es + - es-es + - cs-cz + - fr-fr + - it-it + - da-dk + - nl-nl + - hu-hu + - pl-pl + - pt-pt + - ru-ru + - sv-fi + - sv-se + - tr-tr + +jobs: + Propose_translation: + env: + TARGET_LANG: ${{ inputs.language }} + API_KEY: ${{ secrets.DL_API_KEY }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.ref_name }} + fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: Install Translate Tools + run: | + python -m pip install ovos-translate-plugin-deepl ovos-utils + - name: Run Translate Script + run: python scripts/translate.py + - name: Create Pull Request + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: autotranslate + author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> + title: Proposed ${{ inputs.language }} Translations + body: Translations for review + labels: translation + branch: staging/translation_${{ inputs.language }} + reviewers: emphasize diff --git a/.github/workflows/publish_alpha.yml b/.github/workflows/publish_alpha.yml new file mode 100644 index 0000000..fb96c28 --- /dev/null +++ b/.github/workflows/publish_alpha.yml @@ -0,0 +1,105 @@ +# This workflow will generate a distribution and upload it to PyPI + +name: Publish Alpha Build ...aX +on: + push: + branches: + - dev + paths-ignore: + - 'version.py' + - '.github/**' + - '.gitignore' + - 'LICENSE' + - 'CHANGELOG.md' + - 'MANIFEST.in' + - 'README.md' + - 'scripts/**' + workflow_dispatch: + +jobs: + autotranslate: + runs-on: ubuntu-latest + env: + API_KEY: ${{secrets.DL_API_KEY}} + steps: + - uses: actions/checkout@v3 + with: + ref: dev + fetch-depth: 0 + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + locales: + - 'locale/en-us/**' + - 'dialog/en-us/**' + - 'vocab/en-us/**' + - name: Setup Python + if: steps.filter.outputs.locales == 'true' + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: Auto Translate + if: steps.filter.outputs.locales == 'true' + run: | + python -m pip install ovos-translate-plugin-deepl ovos-utils + python scripts/translate.py + - name: Commit to dev + if: steps.filter.outputs.locales == 'true' + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: autotranslate + branch: dev + + build_and_publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: dev + fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: Install Build Tools + run: | + python -m pip install build wheel + - name: Increment Version + run: | + VER=$(python setup.py --version) + python scripts/bump_alpha.py + - name: "Generate release changelog" + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + id: changelog + - name: Commit to dev + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Increment Version + branch: dev + - name: version + run: echo "::set-output name=version::$(python setup.py --version)" + id: version + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token + with: + tag_name: V${{ steps.version.outputs.version }} + release_name: Release ${{ steps.version.outputs.version }} + body: | + Changes in this Release + ${{ steps.changelog.outputs.changelog }} + draft: false + prerelease: true + commitish: dev + - name: Build Distribution Packages + run: | + python setup.py bdist_wheel + - name: Publish to Test PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{secrets.PYPI_TOKEN}} \ No newline at end of file diff --git a/.github/workflows/publish_build.yml b/.github/workflows/publish_build.yml new file mode 100644 index 0000000..65e168b --- /dev/null +++ b/.github/workflows/publish_build.yml @@ -0,0 +1,83 @@ +# This workflow will generate a distribution and upload it to PyPI + +name: Publish Build Release ..X +on: + workflow_dispatch: + +jobs: + build_and_publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: dev + fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install Build Tools + run: | + python -m pip install build wheel + - name: Install OSM + run: | + pip install ovos-skills-manager~=0.0.10 + - name: Remove alpha (declare stable) + run: | + VER=$(python setup.py --version) + python scripts/remove_alpha.py + - name: Prepare Skillstore metadata + run: | + python scripts/prepare_skillstore.py + python scripts/release_skillstore.py + - name: "Generate release changelog" + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + id: changelog + - name: Commit to dev + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Declare alpha stable + branch: dev + - name: Push dev -> master + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + branch: master + force: true + - name: version + run: echo "::set-output name=version::$(python setup.py --version)" + id: version + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token + with: + tag_name: V${{ steps.version.outputs.version }} + release_name: Release ${{ steps.version.outputs.version }} + body: | + Changes in this Release + ${{ steps.changelog.outputs.changelog }} + draft: false + prerelease: false + - name: Build Distribution Packages + run: | + python setup.py bdist_wheel + - name: Prepare next Build version + run: echo "::set-output name=version::$(python setup.py --version)" + id: alpha + - name: Increment Version ${{ steps.alpha.outputs.version }}Alpha0 + run: | + VER=$(python setup.py --version) + python scripts/bump_build.py + - name: Commit to dev + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Prepare Next Version + branch: dev + - name: Publish to Test PyPI + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{secrets.PYPI_TOKEN}} \ No newline at end of file diff --git a/.github/workflows/publish_major.yml b/.github/workflows/publish_major.yml new file mode 100644 index 0000000..09121b4 --- /dev/null +++ b/.github/workflows/publish_major.yml @@ -0,0 +1,83 @@ +# This workflow will generate a distribution and upload it to PyPI + +name: Publish Major Release X.0.0 +on: + workflow_dispatch: + +jobs: + build_and_publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: dev + fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install Build Tools + run: | + python -m pip install build wheel + - name: Install OSM + run: | + pip install ovos-skills-manager~=0.0.10 + - name: Remove alpha (declare stable) + run: | + VER=$(python setup.py --version) + python scripts/remove_alpha.py + - name: Prepare Skillstore metadata + run: | + python scripts/prepare_skillstore.py + python scripts/release_skillstore.py + - name: "Generate release changelog" + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + id: changelog + - name: Commit to dev + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Declare alpha stable + branch: dev + - name: Push dev -> master + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + branch: master + force: true + - name: version + run: echo "::set-output name=version::$(python setup.py --version)" + id: version + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token + with: + tag_name: V${{ steps.version.outputs.version }} + release_name: Release ${{ steps.version.outputs.version }} + body: | + Changes in this Release + ${{ steps.changelog.outputs.changelog }} + draft: false + prerelease: false + - name: Build Distribution Packages + run: | + python setup.py bdist_wheel + - name: Prepare next Major version + run: echo "::set-output name=version::$(python setup.py --version)" + id: alpha + - name: Increment Version ${{ steps.alpha.outputs.version }}Alpha0 + run: | + VER=$(python setup.py --version) + python scripts/bump_major.py + - name: Commit to dev + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Prepare Next Version + branch: dev + - name: Publish to Test PyPI + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{secrets.PYPI_TOKEN}} \ No newline at end of file diff --git a/.github/workflows/publish_minor.yml b/.github/workflows/publish_minor.yml new file mode 100644 index 0000000..fe98511 --- /dev/null +++ b/.github/workflows/publish_minor.yml @@ -0,0 +1,83 @@ +# This workflow will generate a distribution and upload it to PyPI + +name: Publish Minor Release .X.0 +on: + workflow_dispatch: + +jobs: + build_and_publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: dev + fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install Build Tools + run: | + python -m pip install build wheel + - name: Install OSM + run: | + pip install ovos-skills-manager~=0.0.10 + - name: Remove alpha (declare stable) + run: | + VER=$(python setup.py --version) + python scripts/remove_alpha.py + - name: Prepare Skillstore metadata + run: | + python scripts/prepare_skillstore.py + python scripts/release_skillstore.py + - name: "Generate release changelog" + uses: heinrichreimer/github-changelog-generator-action@v2.3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + id: changelog + - name: Commit to dev + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Declare alpha stable + branch: dev + - name: Push dev -> master + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + branch: master + force: true + - name: version + run: echo "::set-output name=version::$(python setup.py --version)" + id: version + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token + with: + tag_name: V${{ steps.version.outputs.version }} + release_name: Release ${{ steps.version.outputs.version }} + body: | + Changes in this Release + ${{ steps.changelog.outputs.changelog }} + draft: false + prerelease: false + - name: Build Distribution Packages + run: | + python setup.py bdist_wheel + - name: Prepare next Minor version + run: echo "::set-output name=version::$(python setup.py --version)" + id: alpha + - name: Increment Version ${{ steps.alpha.outputs.version }}Alpha0 + run: | + VER=$(python setup.py --version) + python scripts/bump_minor.py + - name: Commit to dev + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Prepare Next Version + branch: dev + - name: Publish to Test PyPI + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{secrets.PYPI_TOKEN}} \ No newline at end of file diff --git a/scripts/bump_alpha.py b/scripts/bump_alpha.py new file mode 100644 index 0000000..e465543 --- /dev/null +++ b/scripts/bump_alpha.py @@ -0,0 +1,18 @@ +import fileinput +from os.path import join, dirname + + +version_file = join(dirname(dirname(__file__)), "version.py") +version_var_name = "VERSION_ALPHA" + +with open(version_file, "r", encoding="utf-8") as v: + for line in v.readlines(): + if line.startswith(version_var_name): + version = int(line.split("=")[-1]) + new_version = int(version) + 1 + +for line in fileinput.input(version_file, inplace=True): + if line.startswith(version_var_name): + print(f"{version_var_name} = {new_version}") + else: + print(line.rstrip('\n')) diff --git a/scripts/bump_build.py b/scripts/bump_build.py new file mode 100644 index 0000000..61099f8 --- /dev/null +++ b/scripts/bump_build.py @@ -0,0 +1,21 @@ +import fileinput +from os.path import join, dirname + + +version_file = join(dirname(dirname(__file__)), "version.py") +version_var_name = "VERSION_BUILD" +alpha_var_name = "VERSION_ALPHA" + +with open(version_file, "r", encoding="utf-8") as v: + for line in v.readlines(): + if line.startswith(version_var_name): + version = int(line.split("=")[-1]) + new_version = int(version) + 1 + +for line in fileinput.input(version_file, inplace=True): + if line.startswith(version_var_name): + print(f"{version_var_name} = {new_version}") + elif line.startswith(alpha_var_name): + print(f"{alpha_var_name} = 0") + else: + print(line.rstrip('\n')) diff --git a/scripts/bump_major.py b/scripts/bump_major.py new file mode 100644 index 0000000..2610fbb --- /dev/null +++ b/scripts/bump_major.py @@ -0,0 +1,27 @@ +import fileinput +from os.path import join, dirname + + +version_file = join(dirname(dirname(__file__)), "version.py") +version_var_name = "VERSION_MAJOR" +minor_var_name = "VERSION_MINOR" +build_var_name = "VERSION_BUILD" +alpha_var_name = "VERSION_ALPHA" + +with open(version_file, "r", encoding="utf-8") as v: + for line in v.readlines(): + if line.startswith(version_var_name): + version = int(line.split("=")[-1]) + new_version = int(version) + 1 + +for line in fileinput.input(version_file, inplace=True): + if line.startswith(version_var_name): + print(f"{version_var_name} = {new_version}") + elif line.startswith(minor_var_name): + print(f"{minor_var_name} = 0") + elif line.startswith(build_var_name): + print(f"{build_var_name} = 0") + elif line.startswith(alpha_var_name): + print(f"{alpha_var_name} = 0") + else: + print(line.rstrip('\n')) diff --git a/scripts/bump_minor.py b/scripts/bump_minor.py new file mode 100644 index 0000000..86dfd9d --- /dev/null +++ b/scripts/bump_minor.py @@ -0,0 +1,24 @@ +import fileinput +from os.path import join, dirname + + +version_file = join(dirname(dirname(__file__)), "version.py") +version_var_name = "VERSION_MINOR" +build_var_name = "VERSION_BUILD" +alpha_var_name = "VERSION_ALPHA" + +with open(version_file, "r", encoding="utf-8") as v: + for line in v.readlines(): + if line.startswith(version_var_name): + version = int(line.split("=")[-1]) + new_version = int(version) + 1 + +for line in fileinput.input(version_file, inplace=True): + if line.startswith(version_var_name): + print(f"{version_var_name} = {new_version}") + elif line.startswith(build_var_name): + print(f"{build_var_name} = 0") + elif line.startswith(alpha_var_name): + print(f"{alpha_var_name} = 0") + else: + print(line.rstrip('\n')) diff --git a/scripts/prepare_skillstore.py b/scripts/prepare_skillstore.py new file mode 100644 index 0000000..143caf9 --- /dev/null +++ b/scripts/prepare_skillstore.py @@ -0,0 +1,76 @@ +from ovos_skills_manager import SkillEntry +from os.path import exists, join, dirname +from shutil import rmtree +import os +from os import makedirs +import json +from ovos_utils.bracket_expansion import expand_parentheses, expand_options + + +branch = "dev" +repo = "skill-ovos-spelling" +author = "OpenVoiceOS" + +url = f"https://github.com/{author}/{repo}@{branch}" + +skill = SkillEntry.from_github_url(url) +tmp_skills = "/tmp/osm_installed_skills" +skill_folder = f"{tmp_skills}/{skill.uuid}" + +base_dir = dirname(dirname(__file__)) +desktop_dir = join(base_dir, "res", "desktop") +android_ui = join(base_dir, "ui", "+android") +makedirs(desktop_dir, exist_ok=True) + +readme = join(base_dir, "README.md") +jsonf = join(desktop_dir, "skill.json") +desktopf = join(desktop_dir, f"{repo}.desktop") +skill_code = join(base_dir, "__init__.py") + +res_folder = join(base_dir, "locale", "en-us") + + +def read_samples(path): + samples = [] + with open(path) as fi: + for _ in fi.read().split("\n"): + if _ and not _.strip().startswith("#"): + samples += expand_options(_) + return samples + +samples = [] +for root, folders, files in os.walk(res_folder): + for f in files: + if f.endswith(".intent"): + samples += read_samples(join(root, f)) +skill._data["examples"] = list(set(samples)) + +has_android = exists(android_ui) +with open(skill_code) as f: + has_homescreen = f"{repo}.{author}.home" in f.read() + +if not exists(readme): + with open(readme, "w") as f: + f.write(skill.generate_readme()) + +if has_homescreen and not exists(desktopf): + with open(desktopf, "w") as f: + f.write(skill.desktop_file) + +if not exists(jsonf): + data = skill.json + with open(jsonf, "w") as f: + if not has_android or not has_homescreen: + data.pop("android") + if not has_homescreen: + data.pop("desktop") + data["desktopFile"] = False +else: + with open(jsonf) as f: + data = json.load(f) + +# set dev branch +data["branch"] = "dev" + +with open(jsonf, "w") as f: + json.dump(data, f, indent=4) diff --git a/scripts/release_skillstore.py b/scripts/release_skillstore.py new file mode 100644 index 0000000..a176d89 --- /dev/null +++ b/scripts/release_skillstore.py @@ -0,0 +1,41 @@ +import json +from os.path import join, dirname + +base_dir = dirname(dirname(__file__)) + + +def get_version(): + """ Find the version of the package""" + version_file = join(base_dir, 'version.py') + major, minor, build, alpha = (None, None, None, None) + with open(version_file) as f: + for line in f: + if 'VERSION_MAJOR' in line: + major = line.split('=')[1].strip() + elif 'VERSION_MINOR' in line: + minor = line.split('=')[1].strip() + elif 'VERSION_BUILD' in line: + build = line.split('=')[1].strip() + elif 'VERSION_ALPHA' in line: + alpha = line.split('=')[1].strip() + + if ((major and minor and build and alpha) or + '# END_VERSION_BLOCK' in line): + break + version = f"{major}.{minor}.{build}" + if alpha and int(alpha) > 0: + version += f"a{alpha}" + return version + + +desktop_dir = join(base_dir, "res", "desktop") + +jsonf = join(desktop_dir, "skill.json") + +with open(jsonf) as f: + data = json.load(f) + +data["branch"] = "v" + get_version() + +with open(jsonf, "w") as f: + json.dump(data, f, indent=4) diff --git a/scripts/remove_alpha.py b/scripts/remove_alpha.py new file mode 100644 index 0000000..fca7342 --- /dev/null +++ b/scripts/remove_alpha.py @@ -0,0 +1,13 @@ +import fileinput +from os.path import join, dirname + + +version_file = join(dirname(dirname(__file__)), "version.py") + +alpha_var_name = "VERSION_ALPHA" + +for line in fileinput.input(version_file, inplace=True): + if line.startswith(alpha_var_name): + print(f"{alpha_var_name} = 0") + else: + print(line.rstrip('\n')) diff --git a/scripts/translate.py b/scripts/translate.py new file mode 100644 index 0000000..43b0687 --- /dev/null +++ b/scripts/translate.py @@ -0,0 +1,157 @@ +from os.path import dirname, join, isdir, exists +from pathlib import Path +import shutil +import os +import re +from ovos_utils.bracket_expansion import expand_options +from ovos_translate_plugin_deepl import DeepLTranslator + + +API_KEY = os.getenv("API_KEY") +if not API_KEY: + raise ValueError + +single_lang = os.getenv("TARGET_LANG") +target_langs = (single_lang,) if single_lang else ("de-de", + "ca-es", + "cs-cz", + "da-dk", + "es-es", + "fr-fr", + "hu-hu", + "it-it", + "nl-nl", + "pl-pl", + "pt-pt", + "ru-ru", + "sv-fi", + "sv-se") + + +base_folder = dirname(dirname(__file__)) +res_folder = join(base_folder, "locale") + +# old structure +old_voc_folder = join(base_folder, "vocab") +old_dialog_folder = join(base_folder, "dialog") +old_res_folder = [old_voc_folder, old_dialog_folder] + +src_lang="en-us" +src_files={} +# note: regex/namedvalues are just copied, this cant be auto translated reliably +ext = [".voc", ".dialog", ".intent", ".entity", ".rx", ".value", ".word"] +untranslated = [".rx", ".value", ".entity"] + +tx = DeepLTranslator({"api_key": API_KEY}) + + +def file_location(f: str, base: str) -> bool: + for root, dirs, files in os.walk(base): + for file in files: + if f == file: + return join(root, file) + return None + +def translate(lines: list, target_lang: str) -> list: + translations = [] + for line in lines: + replacements = dict() + for num, var in enumerate(re.findall(r"(?:{{|{)[ a-zA-Z0-9_]*(?:}}|})", line)): + line = line.replace(var, f'@{num}', 1) + replacements[f'@{num}'] = var + try: + translated = tx.translate(line, target=target_lang, source=src_lang) + except Exception as e: + continue + for num, var in replacements.items(): + translated = translated.replace(num, var) + translations.append(translated) + + return translations + + +def entities(file: str) -> set: + vars = set() + if not exists(file): + return vars + + lines = get_lines(file) + for line in lines: + for var in re.findall(r"(?:{{|{)[ a-zA-Z0-9_]*(?:}}|})", line): + vars.add(var) + return vars + + +def get_lines(file: str): + with open(file, "r") as f: + # entity files often include #-placeholder + if file.endswith(".entity"): + lines = [exp for l in f.read().split("\n") for exp + in expand_options(l) if l] + else: + lines = [exp for l in f.read().split("\n") for exp + in expand_options(l) if l and not l.startswith("#")] + return lines + + +def migrate_locale(folder): + for lang in os.listdir(folder): + path = join(folder, lang) + for root, dirs, files in os.walk(path): + for file in files: + if file_location(file, join(res_folder, lang)) is None: + rel_path = root.replace(folder, "").lstrip("/") + new_path = join(res_folder, rel_path) + os.makedirs(new_path, exist_ok=True) + shutil.move(join(root, file), + join(new_path, file)) + shutil.rmtree(path) + shutil.rmtree(folder) + + +for folder in old_res_folder: + if not isdir(folder): + continue + migrate_locale(folder) + +src_folder = join(res_folder, src_lang) +for root, dirs, files in os.walk(src_folder): + if src_lang not in root: + continue + for f in files: + if any(f.endswith(e) for e in ext): + file_path = join(root, f) + rel_path = file_path.replace(src_folder, "").lstrip("/") + src_files[rel_path] = file_path + +for lang in target_langs: + # service cant translate + if not tx.get_langcode(lang): + continue + for rel_path, src in src_files.items(): + filename = Path(rel_path).name + dst = file_location(filename, join(res_folder, lang)) or \ + join(res_folder, lang, rel_path) + if entities(src) != entities(dst): + if exists(dst): + os.remove(dst) + elif not exists(dst): + pass + else: + continue + os.makedirs(dirname(dst), exist_ok=True) + + lines = get_lines(src) + if any(filename.endswith(e) for e in untranslated): + tx_lines = lines + is_translated = False + else: + tx_lines = translate(lines, lang) + is_translated = True + if tx_lines: + tx_lines = list(set(tx_lines)) + with open(dst, "w") as f: + if is_translated: + f.write(f"# auto translated from {src_lang} to {lang}\n") + for translated in set(tx_lines): + f.write(translated + "\n") diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..93434e8 --- /dev/null +++ b/setup.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +import os +from setuptools import setup +from os import walk, path + +URL = "https://github.com/OpenVoiceOS/skill-ovos-spelling" +SKILL_CLAZZ = "SpellingSkill" # needs to match __init__.py class name +PYPI_NAME = "ovos-skill-spelling" # pip install PYPI_NAME + +# below derived from github url to ensure standard skill_id +SKILL_AUTHOR, SKILL_NAME = URL.split(".com/")[-1].split("/") +SKILL_PKG = SKILL_NAME.lower().replace('-', '_') +PLUGIN_ENTRY_POINT = f'{SKILL_NAME.lower()}.{SKILL_AUTHOR.lower()}={SKILL_PKG}:{SKILL_CLAZZ}' +# skill_id=package_name:SkillClass + + +def find_resource_files(): + resource_base_dirs = ("locale", "ui", "vocab", "dialog", "regex", "skill") + base_dir = path.dirname(__file__) + package_data = ["*.json"] + for res in resource_base_dirs: + if path.isdir(path.join(base_dir, res)): + for (directory, _, files) in walk(path.join(base_dir, res)): + if files: + package_data.append( + path.join(directory.replace(base_dir, "").lstrip('/'), + '*')) + return package_data + + +with open("README.md", "r") as f: + long_description = f.read() + + +def get_version(): + """ Find the version of this skill""" + version_file = os.path.join(os.path.dirname(__file__), 'version.py') + major, minor, build, alpha = (None, None, None, None) + with open(version_file) as f: + for line in f: + if 'VERSION_MAJOR' in line: + major = line.split('=')[1].strip() + elif 'VERSION_MINOR' in line: + minor = line.split('=')[1].strip() + elif 'VERSION_BUILD' in line: + build = line.split('=')[1].strip() + elif 'VERSION_ALPHA' in line: + alpha = line.split('=')[1].strip() + + if ((major and minor and build and alpha) or + '# END_VERSION_BLOCK' in line): + break + version = f"{major}.{minor}.{build}" + if int(alpha): + version += f"a{alpha}" + return version + + +setup( + name=PYPI_NAME, + version=get_version(), + long_description=long_description, + url=URL, + author=SKILL_AUTHOR, + description='OVOS spelling skill', + author_email='jarbasai@mailfence.com', + license='Apache-2.0', + package_dir={SKILL_PKG: ""}, + package_data={SKILL_PKG: find_resource_files()}, + packages=[SKILL_PKG], + include_package_data=True, + keywords='ovos skill plugin', + entry_points={'ovos.plugin.skill': PLUGIN_ENTRY_POINT} +) diff --git a/version.py b/version.py new file mode 100644 index 0000000..ea4e38a --- /dev/null +++ b/version.py @@ -0,0 +1,6 @@ +# START_VERSION_BLOCK +VERSION_MAJOR = 0 +VERSION_MINOR = 0 +VERSION_BUILD = 0 +VERSION_ALPHA = 1 +# END_VERSION_BLOCK