From 96e06baf781d59c17ecf7fe5a6a1cdb3031ebfd5 Mon Sep 17 00:00:00 2001 From: David Huggins-Daines Date: Thu, 28 Dec 2023 12:33:09 -0500 Subject: [PATCH 01/10] fix: switch to scikit-build-core for python 3.12 and editable --- cython/CMakeLists.txt | 22 +++++++++++-------- pyproject.toml | 51 +++++++++++++++++++++++++++++++++++++------ setup.cfg | 27 ----------------------- setup.py | 9 -------- 4 files changed, 57 insertions(+), 52 deletions(-) delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/cython/CMakeLists.txt b/cython/CMakeLists.txt index 26619f564..daa4f7643 100644 --- a/cython/CMakeLists.txt +++ b/cython/CMakeLists.txt @@ -1,14 +1,19 @@ -find_package(PythonExtensions REQUIRED) -find_package(Python COMPONENTS Interpreter Development) -find_package(Cython) +find_package(Python COMPONENTS Interpreter Development.Module REQUIRED) +find_program(CYTHON "cython") if(NOT USE_INSTALLED_POCKETSPHINX) set_property(TARGET pocketsphinx PROPERTY POSITION_INDEPENDENT_CODE on) endif() -add_cython_target(_pocketsphinx _pocketsphinx.pyx) -add_library(_pocketsphinx MODULE ${_pocketsphinx}) -target_link_libraries(_pocketsphinx pocketsphinx) +add_custom_command( + OUTPUT _pocketsphinx.c + DEPENDS _pocketsphinx.pyx + VERBATIM + COMMAND "${CYTHON}" "${CMAKE_CURRENT_SOURCE_DIR}/_pocketsphinx.pyx" --output-file + "${CMAKE_CURRENT_BINARY_DIR}/_pocketsphinx.c") + +python_add_library(_pocketsphinx MODULE "${CMAKE_CURRENT_BINARY_DIR}/_pocketsphinx.c" WITH_SOABI) +target_link_libraries(_pocketsphinx PRIVATE pocketsphinx) target_include_directories( _pocketsphinx PRIVATE ${PYTHON_INCLUDE_DIR} _pocketsphinx PRIVATE ${CMAKE_BINARY_DIR} @@ -18,8 +23,7 @@ target_include_directories( _pocketsphinx INTERFACE ${CMAKE_SOURCE_DIR}/include _pocketsphinx INTERFACE ${CMAKE_BINARY_DIR}/include ) -python_extension_module(_pocketsphinx) -install(TARGETS _pocketsphinx LIBRARY DESTINATION cython/pocketsphinx) +install(TARGETS _pocketsphinx LIBRARY DESTINATION pocketsphinx) if(NOT USE_INSTALLED_POCKETSPHINX) - install(DIRECTORY ${PROJECT_SOURCE_DIR}/model DESTINATION cython/pocketsphinx) + install(DIRECTORY ${PROJECT_SOURCE_DIR}/model DESTINATION pocketsphinx) endif() diff --git a/pyproject.toml b/pyproject.toml index cd8964fa6..a8731358a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,38 @@ [build-system] requires = [ - "wheel", - "setuptools>=45,<64", - "scikit-build~=0.15", - "Cython", - "cmake", - "ninja" + "scikit-build-core", + "Cython" ] -build-backend = "setuptools.build_meta" +build-backend = "scikit_build_core.build" + +[project] +name = "pocketsphinx" +version = "5.0.3" +description = "Official Python bindings for PocketSphinx" +readme = "cython/README.md" +authors = [ + {name = "David Huggins-Daines", email = "dhdaines@gmail.com"} +] +keywords = ["asr", "speech"] +classifiers = [ + "Development Status :: 6 - Mature", + "Programming Language :: C", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", +] + +[project.urls] +Homepage = "https://github.com/cmusphinx/pocketsphinx" +Documentation = "https://pocketsphinx.readthedocs.io/en/latest/" +Repository = "https://github.com/cmusphinx/pocketsphinx.git" +Issues = "https://github.com/cmusphinx/pocketsphinx/issues" + [tool.cibuildwheel] # Build the versions found in Ubuntu LTS, the stable PyPy, and 3.10 # everywhere else @@ -22,3 +47,15 @@ build = [ before-build = "rm -rf _skbuild" # PyPy builds are broken on Windows, and skip 32-bit and musl skip = ["*musl*", "*_i686", "*-win32", "pp*win*"] + +[tool.isort] +profile = "black" + +[tool.flake8] +extend-ignore = "E203" +max-line-length = "88" + +[tool.scikit-build] +cmake.verbose = true +logging.level = "INFO" +wheel.packages = ["cython/pocketsphinx"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index df41a2d51..000000000 --- a/setup.cfg +++ /dev/null @@ -1,27 +0,0 @@ -[metadata] -name = pocketsphinx -version = 5.0.2 -description = Official Python bindings for PocketSphinx -long_description = file: cython/README.md -long_description_content_type = text/markdown -author = David Huggins-Daines -author_email = dhdaines@gmail.com -license = MIT -platforms = any -url = https://github.com/cmusphinx/pocketsphinx -project_urls = - Source = https://github.com/cmusphinx/pocketsphinx - Tracker = https://github.com/cmusphinx/pocketsphinx/issues -keywords = asr, speech -classifiers = - Development Status :: 3 - Alpha - Programming Language :: C - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.7 - License :: OSI Approved :: MIT License - Operating System :: OS Independent -[isort] -profile=black -[flake8] -extend-ignore = E203 -max-line-length = 88 diff --git a/setup.py b/setup.py deleted file mode 100644 index 73dabc6b7..000000000 --- a/setup.py +++ /dev/null @@ -1,9 +0,0 @@ -from skbuild import setup -from setuptools import find_packages - -setup( - packages=find_packages('cython', exclude=["test"]), - package_dir={"": "cython"}, - cmake_languages=["C"], - install_requires=["sounddevice"], -) From 4ed00036b4e9aaced055d5feaa551aec9cc0f9f5 Mon Sep 17 00:00:00 2001 From: David Huggins-Daines Date: Thu, 28 Dec 2023 12:33:26 -0500 Subject: [PATCH 02/10] fix: use importlib to find model path --- cython/pocketsphinx/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cython/pocketsphinx/__init__.py b/cython/pocketsphinx/__init__.py index c7e009e00..28dd97ab4 100644 --- a/cython/pocketsphinx/__init__.py +++ b/cython/pocketsphinx/__init__.py @@ -33,6 +33,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import collections +import importlib.util import os import signal from contextlib import contextmanager @@ -81,7 +82,10 @@ def get_model_path(subpath=None): """ model_path = pocketsphinx._ps_default_modeldir() if model_path is None: - model_path = os.path.join(os.path.dirname(__file__), "model") + # Use importlib to find things (so editable installs work) + model_path = importlib.util.find_spec( + "pocketsphinx.model" + ).submodule_search_locations[0] if subpath is not None: return os.path.join(model_path, subpath) else: @@ -181,6 +185,7 @@ class AudioFile(Pocketsphinx): simple. """ + def __init__(self, audio_file=None, **kwargs): signal.signal(signal.SIGINT, self.stop) @@ -239,6 +244,7 @@ def __init__(self, **kwargs): try: import sounddevice + assert sounddevice except Exception as e: # In case PortAudio is not present, for instance From 42919d26a6dbebc948dee00a8772e860c9951812 Mon Sep 17 00:00:00 2001 From: David Huggins-Daines Date: Thu, 28 Dec 2023 12:34:11 -0500 Subject: [PATCH 03/10] feat: switch to tox for tests --- .github/workflows/tests.yml | 43 ++++++++++++++++++++++++++----------- .gitignore | 1 + .readthedocs.yml | 2 +- requirements.dev.txt | 6 ------ tox.ini | 21 ++++++++++++++++++ 5 files changed, 53 insertions(+), 20 deletions(-) delete mode 100644 requirements.dev.txt create mode 100644 tox.ini diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f88d36572..4e191dd5f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,19 +33,36 @@ jobs: - name: Run tests run: | cmake --build build --target check - pytest: - runs-on: ubuntu-latest + python-tests: + name: test with ${{ matrix.py }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + py: + - "3.12" + - "3.11" + - "3.10" + - "3.9" + - "3.8" + os: + - ubuntu-latest + - macos-latest + - windows-latest steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Install - run: | - sudo apt-get install sox - python -m pip install --upgrade pip - pip install -r requirements.dev.txt - pip install . - - name: Run tests - run: pytest + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Setup python for test ${{ matrix.py }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.py }} + - name: Install tox + run: python -m pip install tox-gh>=1.2 + - name: Setup test suite + run: tox -vv --notest + - name: Run test suite + run: tox --skip-pkg-install pytest-editable: runs-on: ubuntu-latest steps: @@ -55,7 +72,7 @@ jobs: run: | sudo apt-get install sox python -m pip install --upgrade pip - pip install -r requirements.dev.txt + pip install pytest memory_profiler pip install -e . - name: Run tests run: pytest diff --git a/.gitignore b/.gitignore index 2a70e1b52..6d2dbdd2d 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ CTestTestfile.cmake DartConfiguration.tcl cmake_install.cmake +.tox diff --git a/.readthedocs.yml b/.readthedocs.yml index 0dcb2b426..d764d78e6 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,7 +3,7 @@ version: 2 build: os: ubuntu-22.04 tools: - python: "3.9" + python: "3.10" sphinx: configuration: docs/source/conf.py diff --git a/requirements.dev.txt b/requirements.dev.txt deleted file mode 100644 index 7557df8da..000000000 --- a/requirements.dev.txt +++ /dev/null @@ -1,6 +0,0 @@ -cmake~=3.22.0 -scikit-build~=0.15.0 -Cython~=0.29.0 -setuptools>=45.0.0 -pytest~=7.0.0 -memory-profiler==0.60.0 diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..08ce4b633 --- /dev/null +++ b/tox.ini @@ -0,0 +1,21 @@ +[tox] +env_list = py{38,39,310,311,312} +minversion = 4.11.4 + +[testenv] +description = run the tests with pytest +package = wheel +wheel_build_env = .pkg +deps = + pytest>=6 + memory_profiler +commands = + pytest {tty:--color=yes} {posargs} + +[gh] +python = + 3.12 = py312 + 3.11 = py311 + 3.10 = py310 + 3.9 = py39 + 3.8 = py38 From ae152c2401c1eb52d29d07345ab588819bb58b75 Mon Sep 17 00:00:00 2001 From: David Huggins-Daines Date: Thu, 28 Dec 2023 12:39:12 -0500 Subject: [PATCH 04/10] fix(tests): install sox --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4e191dd5f..5bc797745 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -57,6 +57,8 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.py }} + - name: Install sox + run: sudo apt-get install sox - name: Install tox run: python -m pip install tox-gh>=1.2 - name: Setup test suite From 541c916f7fd2419cd17565d645b7621e47e0cfe6 Mon Sep 17 00:00:00 2001 From: David Huggins-Daines Date: Thu, 28 Dec 2023 13:00:04 -0500 Subject: [PATCH 05/10] fix(tests): try not to test where no sox --- .github/workflows/tests.yml | 1 + cython/test/endpointer_test.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5bc797745..8fb0559b5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -58,6 +58,7 @@ jobs: with: python-version: ${{ matrix.py }} - name: Install sox + if: ${{ matrix.os == "ubuntu-latest" }} run: sudo apt-get install sox - name: Install tox run: python -m pip install tox-gh>=1.2 diff --git a/cython/test/endpointer_test.py b/cython/test/endpointer_test.py index 8e09a4f92..18bee77c9 100644 --- a/cython/test/endpointer_test.py +++ b/cython/test/endpointer_test.py @@ -243,11 +243,14 @@ def srtest(self, sample_rate): idx += 1 def testEndpointer(self): - set_loglevel("INFO") - # 8000, 44100, 48000 give slightly different results unfortunately - for sample_rate in 11025, 16000, 22050, 32000: - print(sample_rate) - self.srtest(sample_rate) + try: + set_loglevel("INFO") + # 8000, 44100, 48000 give slightly different results unfortunately + for sample_rate in 11025, 16000, 22050, 32000: + print(sample_rate) + self.srtest(sample_rate) + except OSError as err: + self.skipTest("sox not installed: %s" % err) if __name__ == "__main__": From 64d9f31d3fab6adea9608dea213945e7c3881cc5 Mon Sep 17 00:00:00 2001 From: David Huggins-Daines Date: Thu, 28 Dec 2023 13:04:29 -0500 Subject: [PATCH 06/10] fix: WTF, github actions! --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8fb0559b5..e0892f8df 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -58,7 +58,7 @@ jobs: with: python-version: ${{ matrix.py }} - name: Install sox - if: ${{ matrix.os == "ubuntu-latest" }} + if: ${{ startsWith(matrix.os, 'ubuntu') }} run: sudo apt-get install sox - name: Install tox run: python -m pip install tox-gh>=1.2 From 48aaac18350cf156cca23135ff331449b93cc7b0 Mon Sep 17 00:00:00 2001 From: David Huggins-Daines Date: Thu, 28 Dec 2023 13:18:25 -0500 Subject: [PATCH 07/10] tests: be super hecking verbose to diagnose gh actions --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 08ce4b633..f1d33e54d 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ deps = pytest>=6 memory_profiler commands = - pytest {tty:--color=yes} {posargs} + pytest {tty:--color=yes} -vvv -s --full-trace {posargs} [gh] python = From 29a66540155e3ccd70a74844eebc1facd3d049b3 Mon Sep 17 00:00:00 2001 From: David Huggins-Daines Date: Thu, 28 Dec 2023 13:23:59 -0500 Subject: [PATCH 08/10] tests: frob yaml again --- .github/workflows/tests.yml | 12 +++++------- tox.ini | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e0892f8df..bb31222df 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install run: | sudo apt-get install sox ninja-build @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install run: | sudo apt-get install sox ninja-build @@ -37,7 +37,7 @@ jobs: name: test with ${{ matrix.py }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: - fail-fast: false + fail-fast: true matrix: py: - "3.12" @@ -50,9 +50,7 @@ jobs: - macos-latest - windows-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Setup python for test ${{ matrix.py }} uses: actions/setup-python@v4 with: @@ -70,7 +68,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install run: | sudo apt-get install sox diff --git a/tox.ini b/tox.ini index f1d33e54d..ef9782e57 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ deps = pytest>=6 memory_profiler commands = - pytest {tty:--color=yes} -vvv -s --full-trace {posargs} + pytest {tty:--color=yes} -s -k lm_test --full-trace {posargs} [gh] python = From 563b6a8e734354c5cf9cc983c938cb18959aa35e Mon Sep 17 00:00:00 2001 From: David Huggins-Daines Date: Thu, 28 Dec 2023 14:07:55 -0500 Subject: [PATCH 09/10] fix: buffer overflow in NGramModel.prob --- cython/_pocketsphinx.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cython/_pocketsphinx.pyx b/cython/_pocketsphinx.pyx index 3e3ecead9..15ded6243 100644 --- a/cython/_pocketsphinx.pyx +++ b/cython/_pocketsphinx.pyx @@ -610,7 +610,7 @@ cdef class NGramModel: cdef const char **cwords cdef int prob bwords = [w.encode("utf-8") for w in words] - cwords = malloc(len(bwords)) + cwords = malloc(len(bwords) * sizeof(char *)) for i, w in enumerate(bwords): cwords[i] = w prob = ngram_prob(self.lm, cwords, len(words)) From 00d7736cdee38e44f0c00efbb555f78247bc13eb Mon Sep 17 00:00:00 2001 From: David Huggins-Daines Date: Thu, 28 Dec 2023 14:09:15 -0500 Subject: [PATCH 10/10] ci: run all tests again --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ef9782e57..08ce4b633 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ deps = pytest>=6 memory_profiler commands = - pytest {tty:--color=yes} -s -k lm_test --full-trace {posargs} + pytest {tty:--color=yes} {posargs} [gh] python =