diff --git a/CMakeLists.txt b/CMakeLists.txt index 9999dc3860..89c8bd66b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,7 +59,7 @@ endif() option(BUILD_SOX "Build libsox statically" ON) option(BUILD_KALDI "Build kaldi statically" ON) option(BUILD_RNNT "Enable RNN transducer" ON) -option(BUILD_KENLM "Build KenLM statically" ON) +option(BUILD_CTC_DECODER "Build Flashlight CTC decoder" ON) option(BUILD_TORCHAUDIO_PYTHON_EXTENSION "Build Python extension" OFF) option(USE_CUDA "Enable CUDA support" OFF) option(USE_ROCM "Enable ROCM support" OFF) diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt index 6c63ed7428..197e35db9b 100644 --- a/third_party/CMakeLists.txt +++ b/third_party/CMakeLists.txt @@ -26,7 +26,7 @@ set_property(GLOBAL PROPERTY TORCHAUDIO_THIRD_PARTIES "${TORCHAUDIO_THIRD_PARTIE ################################################################################ # KenLM ################################################################################ -if (BUILD_KENLM) +if (BUILD_CTC_DECODER) add_subdirectory(zlib) add_subdirectory(bzip2) add_subdirectory(lzma) diff --git a/third_party/boost/CMakeLists.txt b/third_party/boost/CMakeLists.txt index 367cb43523..2f7ff591e2 100644 --- a/third_party/boost/CMakeLists.txt +++ b/third_party/boost/CMakeLists.txt @@ -1,17 +1,18 @@ include(ExternalProject) -set(INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../install) +set(INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../install/include) set(ARCHIVE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../archives) -ExternalProject_Add(boost_ + +ExternalProject_Add(boost PREFIX ${CMAKE_CURRENT_BINARY_DIR} DOWNLOAD_DIR ${ARCHIVE_DIR} URL https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz URL_HASH SHA256=94ced8b72956591c4775ae2207a9763d3600b30d9d7446562c552f0a14a63be7 BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND ./bootstrap.sh --with-libraries=thread --prefix=${INSTALL_DIR} - BUILD_COMMAND ./b2 link=static - INSTALL_COMMAND ./b2 link=static install + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory boost ${INCLUDE_DIR}/boost DOWNLOAD_NO_PROGRESS ON LOG_DOWNLOAD ON LOG_UPDATE ON @@ -21,8 +22,3 @@ ExternalProject_Add(boost_ LOG_MERGED_STDOUTERR ON LOG_OUTPUT_ON_FAILURE ON ) - -add_library(boost INTERFACE) -add_dependencies(boost boost_) -target_include_directories(boost INTERFACE ${INSTALL_DIR}/include) -target_link_libraries(boost INTERFACE ${INSTALL_DIR}/lib/libboost_thread.a) diff --git a/third_party/bzip2/CMakeLists.txt b/third_party/bzip2/CMakeLists.txt index 70ba3ceb24..dfccfc7b08 100644 --- a/third_party/bzip2/CMakeLists.txt +++ b/third_party/bzip2/CMakeLists.txt @@ -3,13 +3,20 @@ include(ExternalProject) set(INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../install) set(ARCHIVE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../archives) -ExternalProject_Add(bzip2_ +set( + BZIP_LIBRARIES + ${INSTALL_DIR}/lib/libbz2.a + ) + +ExternalProject_Add(bzip2- PREFIX ${CMAKE_CURRENT_BINARY_DIR} DOWNLOAD_DIR ${ARCHIVE_DIR} URL https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz URL_HASH SHA256=ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269 + BUILD_BYPRODUCTS ${BZIP_LIBRARIES} BUILD_IN_SOURCE 1 CONFIGURE_COMMAND "" + BUILD_COMMAND make VERBOSE=1 "CFLAGS=-fPIC -fvisibility=hidden -Wall -Winline -O2 -g -D_FILE_OFFSET_BITS=64" INSTALL_COMMAND make install PREFIX=${INSTALL_DIR} DOWNLOAD_NO_PROGRESS ON LOG_DOWNLOAD ON @@ -22,6 +29,6 @@ ExternalProject_Add(bzip2_ ) add_library(bzip2 INTERFACE) -add_dependencies(bzip2 bzip2_) +add_dependencies(bzip2 bzip2-) target_include_directories(bzip2 INTERFACE ${INSTALL_DIR}/include) -target_link_libraries(bzip2 INTERFACE ${INSTALL_DIR}/lib/libbz2.a) +target_link_libraries(bzip2 INTERFACE ${BZIP_LIBRARIES}) diff --git a/third_party/lzma/CMakeLists.txt b/third_party/lzma/CMakeLists.txt index ee89e67816..8da79c5af2 100644 --- a/third_party/lzma/CMakeLists.txt +++ b/third_party/lzma/CMakeLists.txt @@ -10,15 +10,22 @@ set(ARCHIVE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../archives) set(envs "PKG_CONFIG_PATH=${INSTALL_DIR}/lib/pkgconfig" "LDFLAGS=-L${INSTALL_DIR}/lib $ENV{LDFLAGS}" - "CFLAGS=-I${INSTALL_DIR}/include -fvisibility=hidden $ENV{CFLAGS}" + "CFLAGS=-I${INSTALL_DIR}/include -fvisibility=hidden -fPIC $ENV{CFLAGS}" ) -ExternalProject_Add(lzma_ +set( + LZMA_LIBRARIES + ${INSTALL_DIR}/lib/liblzma.a + ) + +ExternalProject_Add(lzma- PREFIX ${CMAKE_CURRENT_BINARY_DIR} DOWNLOAD_DIR ${ARCHIVE_DIR} URL https://tukaani.org/xz/xz-5.2.5.tar.gz URL_HASH SHA256=f6f4910fd033078738bd82bfba4f49219d03b17eb0794eb91efbae419f4aba10 - CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/lzma_/configure --prefix=${INSTALL_DIR} --disable-xz --disable-xzdec --disable-lzmadec --disable-lzmainfo --disable-lzma-links --disable-scripts --disable-doc --enable-static --disable-shared + BUILD_BYPRODUCTS ${LZMA_LIBRARIES} + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/lzma-/configure --prefix=${INSTALL_DIR} --disable-xz --disable-xzdec --disable-lzmadec --disable-lzmainfo --disable-lzma-links --disable-scripts --disable-doc --enable-static --disable-shared + BUILD_COMMAND ${CMAKE_COMMAND} -E env ${envs} make VERBOSE=1 DOWNLOAD_NO_PROGRESS ON LOG_DOWNLOAD ON LOG_UPDATE ON @@ -31,6 +38,6 @@ ExternalProject_Add(lzma_ add_library(lzma INTERFACE) -add_dependencies(lzma lzma_) +add_dependencies(lzma lzma-) target_include_directories(lzma INTERFACE ${INSTALL_DIR}/include) -target_link_libraries(lzma INTERFACE ${INSTALL_DIR}/lib/liblzma.a) +target_link_libraries(lzma INTERFACE ${LZMA_LIBRARIES}) diff --git a/third_party/zlib/CMakeLists.txt b/third_party/zlib/CMakeLists.txt index aebbec84e4..c305edc161 100644 --- a/third_party/zlib/CMakeLists.txt +++ b/third_party/zlib/CMakeLists.txt @@ -10,17 +10,23 @@ set(ARCHIVE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../archives) set(envs "PKG_CONFIG_PATH=${INSTALL_DIR}/lib/pkgconfig" "LDFLAGS=-L${INSTALL_DIR}/lib $ENV{LDFLAGS}" - "CFLAGS=-I${INSTALL_DIR}/include -fvisibility=hidden $ENV{CFLAGS}" + "CFLAGS=-I${INSTALL_DIR}/include -fvisibility=hidden -fPIC $ENV{CFLAGS}" "prefix=${INSTALL_DIR}" ) -ExternalProject_Add(zlib_ +set( + ZLIB_LIBRARIES + ${INSTALL_DIR}/lib/libz.a + ) + +ExternalProject_Add(zlib- PREFIX ${CMAKE_CURRENT_BINARY_DIR} DOWNLOAD_DIR ${ARCHIVE_DIR} URL https://zlib.net/zlib-1.2.11.tar.gz URL_HASH SHA256=c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1 - CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/zlib_/configure --static - DOWNLOAD_NO_PROGRESS ON + BUILD_BYPRODUCTS ${ZLIB_LIBRARIES} + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env ${envs} ${CMAKE_CURRENT_BINARY_DIR}/src/zlib-/configure --static + BUILD_COMMAND ${CMAKE_COMMAND} -E env ${envs} make VERBOSE=1 LOG_DOWNLOAD ON LOG_UPDATE ON LOG_CONFIGURE ON @@ -31,6 +37,6 @@ ExternalProject_Add(zlib_ ) add_library(zlib INTERFACE) -add_dependencies(zlib zlib_) +add_dependencies(zlib zlib-) target_include_directories(zlib INTERFACE ${INSTALL_DIR}/include) -target_link_libraries(zlib INTERFACE ${INSTALL_DIR}/lib/libz.a) +target_link_libraries(zlib INTERFACE ${ZLIB_LIBRARIES}) diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index fe4331cf93..a9d8592d4d 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -37,7 +37,7 @@ def _get_build(var, default=False): _BUILD_SOX = False if platform.system() == 'Windows' else _get_build("BUILD_SOX", True) _BUILD_KALDI = False if platform.system() == 'Windows' else _get_build("BUILD_KALDI", True) _BUILD_RNNT = _get_build("BUILD_RNNT", True) -_BUILD_KENLM = False if platform.system() == 'Windows' else _get_build("BUILD_KENLM", True) +_BUILD_CTC_DECODER = False if platform.system() == 'Windows' else _get_build("BUILD_CTC_DECODER", True) _USE_ROCM = _get_build("USE_ROCM", torch.cuda.is_available() and torch.version.hip is not None) _USE_CUDA = _get_build("USE_CUDA", torch.cuda.is_available() and torch.version.hip is None) _USE_OPENMP = _get_build("USE_OPENMP", True) and \ @@ -46,10 +46,16 @@ def _get_build(var, default=False): def get_ext_modules(): - return [ + modules = [ Extension(name='torchaudio.lib.libtorchaudio', sources=[]), Extension(name='torchaudio._torchaudio', sources=[]), ] + if _BUILD_CTC_DECODER: + modules.extend([ + Extension(name='torchaudio.lib.libtorchaudio_decoder', sources=[]), + Extension(name='torchaudio._torchaudio_decoder', sources=[]), + ]) + return modules # Based off of @@ -90,7 +96,7 @@ def build_extension(self, ext): f"-DBUILD_SOX:BOOL={'ON' if _BUILD_SOX else 'OFF'}", f"-DBUILD_KALDI:BOOL={'ON' if _BUILD_KALDI else 'OFF'}", f"-DBUILD_RNNT:BOOL={'ON' if _BUILD_RNNT else 'OFF'}", - f"-DBUILD_KENLM:BOOL={'ON' if _BUILD_KENLM else 'OFF'}", + f"-DBUILD_CTC_DECODER:BOOL={'ON' if _BUILD_CTC_DECODER else 'OFF'}", "-DBUILD_TORCHAUDIO_PYTHON_EXTENSION:BOOL=ON", f"-DUSE_ROCM:BOOL={'ON' if _USE_ROCM else 'OFF'}", f"-DUSE_CUDA:BOOL={'ON' if _USE_CUDA else 'OFF'}", diff --git a/torchaudio/csrc/CMakeLists.txt b/torchaudio/csrc/CMakeLists.txt index d4c7b62d71..9c58b08b43 100644 --- a/torchaudio/csrc/CMakeLists.txt +++ b/torchaudio/csrc/CMakeLists.txt @@ -126,6 +126,51 @@ define_library( "${LIBTORCHAUDIO_COMPILE_DEFINITIONS}" ) + +################################################################################ +# libtorchaudio_decoder.so +################################################################################ +if (BUILD_CTC_DECODER) + set( + LIBTORCHAUDIO_DECODER_SOURCES + decoder/src/decoder/Decoder.h + decoder/src/decoder/LexiconDecoder.cpp + decoder/src/decoder/LexiconDecoder.h + decoder/src/decoder/Trie.cpp + decoder/src/decoder/Trie.h + decoder/src/decoder/Utils.cpp + decoder/src/decoder/Utils.h + decoder/src/decoder/lm/KenLM.cpp + decoder/src/decoder/lm/KenLM.h + decoder/src/decoder/lm/LM.h + decoder/src/dictionary/Defines.h + decoder/src/dictionary/Dictionary.cpp + decoder/src/dictionary/Dictionary.h + decoder/src/dictionary/String.cpp + decoder/src/dictionary/String.h + decoder/src/dictionary/System.cpp + decoder/src/dictionary/System.h + decoder/src/dictionary/Utils.cpp + decoder/src/dictionary/Utils.h + ) + set( + LIBTORCHAUDIO_DECODER_DEFINITIONS + BUILD_CTC_DECODER + ) + set( + LIBTORCHAUDIO_DECODER_DEPS + libtorchaudio + kenlm) + define_library( + libtorchaudio_decoder + "${LIBTORCHAUDIO_DECODER_SOURCES}" + "${PROJECT_SOURCE_DIR}" + "${LIBTORCHAUDIO_DECODER_DEPS}" + "${LIBTORCHAUDIO_COMPILE_DEFINITIONS};${LIBTORCHAUDIO_DECODER_DEFINITIONS}" + ) +endif() + +# TODO: Add libtorchaudio_decoder if (APPLE) set(TORCHAUDIO_LIBRARY libtorchaudio CACHE INTERNAL "") else() @@ -195,4 +240,16 @@ if (BUILD_TORCHAUDIO_PYTHON_EXTENSION) libtorchaudio "${LIBTORCHAUDIO_COMPILE_DEFINITIONS}" ) + if(BUILD_CTC_DECODER) + set( + DECODER_EXTENSION_SOURCES + decoder/bindings/pybind.cpp + ) + define_extension( + _torchaudio_decoder + "${DECODER_EXTENSION_SOURCES}" + "libtorchaudio;libtorchaudio_decoder" + "${LIBTORCHAUDIO_DECODER_DEFINITIONS}" + ) + endif() endif() diff --git a/torchaudio/csrc/decoder/bindings/cmake/Buildpybind11.cmake b/torchaudio/csrc/decoder/bindings/cmake/Buildpybind11.cmake deleted file mode 100755 index d891b838ac..0000000000 --- a/torchaudio/csrc/decoder/bindings/cmake/Buildpybind11.cmake +++ /dev/null @@ -1,19 +0,0 @@ -include(ExternalProject) - -set(pybind11_URL https://github.com/pybind/pybind11.git) -set(pybind11_TAG 9a19306fbf30642ca331d0ec88e7da54a96860f9) # release 2.2.4 - -# Download pybind11 -ExternalProject_Add( - pybind11 - PREFIX pybind11 - GIT_REPOSITORY ${pybind11_URL} - GIT_TAG ${pybind11_TAG} - BUILD_IN_SOURCE 0 - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" -) - -ExternalProject_Get_Property(pybind11 SOURCE_DIR) -set(pybind11_INCLUDE_DIR "${SOURCE_DIR}/include") \ No newline at end of file diff --git a/torchaudio/csrc/decoder/bindings/cmake/FindPythonLibsNew.cmake b/torchaudio/csrc/decoder/bindings/cmake/FindPythonLibsNew.cmake deleted file mode 100644 index 919a384ea4..0000000000 --- a/torchaudio/csrc/decoder/bindings/cmake/FindPythonLibsNew.cmake +++ /dev/null @@ -1,198 +0,0 @@ -# - Find python libraries -# This module finds the libraries corresponding to the Python interpreter -# FindPythonInterp provides. -# This code sets the following variables: -# -# PYTHONLIBS_FOUND - have the Python libs been found -# PYTHON_PREFIX - path to the Python installation -# PYTHON_LIBRARIES - path to the python library -# PYTHON_INCLUDE_DIRS - path to where Python.h is found -# PYTHON_MODULE_EXTENSION - lib extension, e.g. '.so' or '.pyd' -# PYTHON_MODULE_PREFIX - lib name prefix: usually an empty string -# PYTHON_SITE_PACKAGES - path to installation site-packages -# PYTHON_IS_DEBUG - whether the Python interpreter is a debug build -# -# Thanks to talljimbo for the patch adding the 'LDVERSION' config -# variable usage. - -#============================================================================= -# Copyright 2001-2009 Kitware, Inc. -# Copyright 2012 Continuum Analytics, Inc. -# -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# * Neither the names of Kitware, Inc., the Insight Software Consortium, -# nor the names of their contributors may be used to endorse or promote -# products derived from this software without specific prior written -# permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#============================================================================= - -# Checking for the extension makes sure that `LibsNew` was found and not just `Libs`. -if(PYTHONLIBS_FOUND AND PYTHON_MODULE_EXTENSION) - return() -endif() - -# Use the Python interpreter to find the libs. -if(PythonLibsNew_FIND_REQUIRED) - find_package(PythonInterp ${PythonLibsNew_FIND_VERSION} REQUIRED) -else() - find_package(PythonInterp ${PythonLibsNew_FIND_VERSION}) -endif() - -if(NOT PYTHONINTERP_FOUND) - set(PYTHONLIBS_FOUND FALSE) - return() -endif() - -# According to http://stackoverflow.com/questions/646518/python-how-to-detect-debug-interpreter -# testing whether sys has the gettotalrefcount function is a reliable, cross-platform -# way to detect a CPython debug interpreter. -# -# The library suffix is from the config var LDVERSION sometimes, otherwise -# VERSION. VERSION will typically be like "2.7" on unix, and "27" on windows. -execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" - "from distutils import sysconfig as s;import sys;import struct; -print('.'.join(str(v) for v in sys.version_info)); -print(sys.prefix); -print(s.get_python_inc(plat_specific=True)); -print(s.get_python_lib(plat_specific=True)); -print(s.get_config_var('SO')); -print(hasattr(sys, 'gettotalrefcount')+0); -print(struct.calcsize('@P')); -print(s.get_config_var('LDVERSION') or s.get_config_var('VERSION')); -print(s.get_config_var('LIBDIR') or ''); -print(s.get_config_var('MULTIARCH') or ''); -" - RESULT_VARIABLE _PYTHON_SUCCESS - OUTPUT_VARIABLE _PYTHON_VALUES - ERROR_VARIABLE _PYTHON_ERROR_VALUE) - -if(NOT _PYTHON_SUCCESS MATCHES 0) - if(PythonLibsNew_FIND_REQUIRED) - message(FATAL_ERROR - "Python config failure:\n${_PYTHON_ERROR_VALUE}") - endif() - set(PYTHONLIBS_FOUND FALSE) - return() -endif() - -# Convert the process output into a list -if(WIN32) - string(REGEX REPLACE "\\\\" "/" _PYTHON_VALUES ${_PYTHON_VALUES}) -endif() -string(REGEX REPLACE ";" "\\\\;" _PYTHON_VALUES ${_PYTHON_VALUES}) -string(REGEX REPLACE "\n" ";" _PYTHON_VALUES ${_PYTHON_VALUES}) -list(GET _PYTHON_VALUES 0 _PYTHON_VERSION_LIST) -list(GET _PYTHON_VALUES 1 PYTHON_PREFIX) -list(GET _PYTHON_VALUES 2 PYTHON_INCLUDE_DIR) -list(GET _PYTHON_VALUES 3 PYTHON_SITE_PACKAGES) -list(GET _PYTHON_VALUES 4 PYTHON_MODULE_EXTENSION) -list(GET _PYTHON_VALUES 5 PYTHON_IS_DEBUG) -list(GET _PYTHON_VALUES 6 PYTHON_SIZEOF_VOID_P) -list(GET _PYTHON_VALUES 7 PYTHON_LIBRARY_SUFFIX) -list(GET _PYTHON_VALUES 8 PYTHON_LIBDIR) -list(GET _PYTHON_VALUES 9 PYTHON_MULTIARCH) - -# Make sure the Python has the same pointer-size as the chosen compiler -# Skip if CMAKE_SIZEOF_VOID_P is not defined -if(CMAKE_SIZEOF_VOID_P AND (NOT "${PYTHON_SIZEOF_VOID_P}" STREQUAL "${CMAKE_SIZEOF_VOID_P}")) - if(PythonLibsNew_FIND_REQUIRED) - math(EXPR _PYTHON_BITS "${PYTHON_SIZEOF_VOID_P} * 8") - math(EXPR _CMAKE_BITS "${CMAKE_SIZEOF_VOID_P} * 8") - message(FATAL_ERROR - "Python config failure: Python is ${_PYTHON_BITS}-bit, " - "chosen compiler is ${_CMAKE_BITS}-bit") - endif() - set(PYTHONLIBS_FOUND FALSE) - return() -endif() - -# The built-in FindPython didn't always give the version numbers -string(REGEX REPLACE "\\." ";" _PYTHON_VERSION_LIST ${_PYTHON_VERSION_LIST}) -list(GET _PYTHON_VERSION_LIST 0 PYTHON_VERSION_MAJOR) -list(GET _PYTHON_VERSION_LIST 1 PYTHON_VERSION_MINOR) -list(GET _PYTHON_VERSION_LIST 2 PYTHON_VERSION_PATCH) - -# Make sure all directory separators are '/' -string(REGEX REPLACE "\\\\" "/" PYTHON_PREFIX ${PYTHON_PREFIX}) -string(REGEX REPLACE "\\\\" "/" PYTHON_INCLUDE_DIR ${PYTHON_INCLUDE_DIR}) -string(REGEX REPLACE "\\\\" "/" PYTHON_SITE_PACKAGES ${PYTHON_SITE_PACKAGES}) - -if(CMAKE_HOST_WIN32) - set(PYTHON_LIBRARY - "${PYTHON_PREFIX}/libs/Python${PYTHON_LIBRARY_SUFFIX}.lib") - - # when run in a venv, PYTHON_PREFIX points to it. But the libraries remain in the - # original python installation. They may be found relative to PYTHON_INCLUDE_DIR. - if(NOT EXISTS "${PYTHON_LIBRARY}") - get_filename_component(_PYTHON_ROOT ${PYTHON_INCLUDE_DIR} DIRECTORY) - set(PYTHON_LIBRARY - "${_PYTHON_ROOT}/libs/Python${PYTHON_LIBRARY_SUFFIX}.lib") - endif() - - # raise an error if the python libs are still not found. - if(NOT EXISTS "${PYTHON_LIBRARY}") - message(FATAL_ERROR "Python libraries not found") - endif() - -else() - if(PYTHON_MULTIARCH) - set(_PYTHON_LIBS_SEARCH "${PYTHON_LIBDIR}/${PYTHON_MULTIARCH}" "${PYTHON_LIBDIR}") - else() - set(_PYTHON_LIBS_SEARCH "${PYTHON_LIBDIR}") - endif() - #message(STATUS "Searching for Python libs in ${_PYTHON_LIBS_SEARCH}") - # Probably this needs to be more involved. It would be nice if the config - # information the python interpreter itself gave us were more complete. - find_library(PYTHON_LIBRARY - NAMES "python${PYTHON_LIBRARY_SUFFIX}" - PATHS ${_PYTHON_LIBS_SEARCH} - NO_DEFAULT_PATH) - - # If all else fails, just set the name/version and let the linker figure out the path. - if(NOT PYTHON_LIBRARY) - set(PYTHON_LIBRARY python${PYTHON_LIBRARY_SUFFIX}) - endif() -endif() - -MARK_AS_ADVANCED( - PYTHON_LIBRARY - PYTHON_INCLUDE_DIR -) - -# We use PYTHON_INCLUDE_DIR, PYTHON_LIBRARY and PYTHON_DEBUG_LIBRARY for the -# cache entries because they are meant to specify the location of a single -# library. We now set the variables listed by the documentation for this -# module. -SET(PYTHON_INCLUDE_DIRS "${PYTHON_INCLUDE_DIR}") -SET(PYTHON_LIBRARIES "${PYTHON_LIBRARY}") -SET(PYTHON_DEBUG_LIBRARIES "${PYTHON_DEBUG_LIBRARY}") - -find_package_message(PYTHON - "Found PythonLibs: ${PYTHON_LIBRARY}" - "${PYTHON_EXECUTABLE}${PYTHON_VERSION}") - -set(PYTHONLIBS_FOUND TRUE) diff --git a/torchaudio/csrc/decoder/bindings/cmake/pybind11Tools.cmake b/torchaudio/csrc/decoder/bindings/cmake/pybind11Tools.cmake deleted file mode 100755 index 0fdacc78d8..0000000000 --- a/torchaudio/csrc/decoder/bindings/cmake/pybind11Tools.cmake +++ /dev/null @@ -1,224 +0,0 @@ -# tools/pybind11Tools.cmake -- Build system for the pybind11 modules -# -# Copyright (c) 2015 Wenzel Jakob -# -# All rights reserved. Use of this source code is governed by a -# BSD-style license that can be found in the LICENSE file. - -cmake_minimum_required(VERSION 2.8.12) - -# Add a CMake parameter for choosing a desired Python version -if(NOT PYBIND11_PYTHON_VERSION) - set(PYBIND11_PYTHON_VERSION "" CACHE STRING "Python version to use for compiling modules") -endif() - -set(Python_ADDITIONAL_VERSIONS 3.7 3.6 3.5 3.4) -find_package(PythonLibsNew ${PYBIND11_PYTHON_VERSION} REQUIRED) - -include(CheckCXXCompilerFlag) -include(CMakeParseArguments) - -if(NOT PYBIND11_CPP_STANDARD AND NOT CMAKE_CXX_STANDARD) - if(NOT MSVC) - check_cxx_compiler_flag("-std=c++14" HAS_CPP14_FLAG) - - if (HAS_CPP14_FLAG) - set(PYBIND11_CPP_STANDARD -std=c++14) - else() - check_cxx_compiler_flag("-std=c++11" HAS_CPP11_FLAG) - if (HAS_CPP11_FLAG) - set(PYBIND11_CPP_STANDARD -std=c++11) - else() - message(FATAL_ERROR "Unsupported compiler -- pybind11 requires C++11 support!") - endif() - endif() - elseif(MSVC) - set(PYBIND11_CPP_STANDARD /std:c++14) - endif() - - set(PYBIND11_CPP_STANDARD ${PYBIND11_CPP_STANDARD} CACHE STRING - "C++ standard flag, e.g. -std=c++11, -std=c++14, /std:c++14. Defaults to C++14 mode." FORCE) -endif() - -# Checks whether the given CXX/linker flags can compile and link a cxx file. cxxflags and -# linkerflags are lists of flags to use. The result variable is a unique variable name for each set -# of flags: the compilation result will be cached base on the result variable. If the flags work, -# sets them in cxxflags_out/linkerflags_out internal cache variables (in addition to ${result}). -function(_pybind11_return_if_cxx_and_linker_flags_work result cxxflags linkerflags cxxflags_out linkerflags_out) - set(CMAKE_REQUIRED_LIBRARIES ${linkerflags}) - check_cxx_compiler_flag("${cxxflags}" ${result}) - if (${result}) - set(${cxxflags_out} "${cxxflags}" CACHE INTERNAL "" FORCE) - set(${linkerflags_out} "${linkerflags}" CACHE INTERNAL "" FORCE) - endif() -endfunction() - -# Internal: find the appropriate link time optimization flags for this compiler -function(_pybind11_add_lto_flags target_name prefer_thin_lto) - if (NOT DEFINED PYBIND11_LTO_CXX_FLAGS) - set(PYBIND11_LTO_CXX_FLAGS "" CACHE INTERNAL "") - set(PYBIND11_LTO_LINKER_FLAGS "" CACHE INTERNAL "") - - if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") - set(cxx_append "") - set(linker_append "") - if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND NOT APPLE) - # Clang Gold plugin does not support -Os; append -O3 to MinSizeRel builds to override it - set(linker_append ";$<$:-O3>") - elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU") - set(cxx_append ";-fno-fat-lto-objects") - endif() - - if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND prefer_thin_lto) - _pybind11_return_if_cxx_and_linker_flags_work(HAS_FLTO_THIN - "-flto=thin${cxx_append}" "-flto=thin${linker_append}" - PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) - endif() - - if (NOT HAS_FLTO_THIN) - _pybind11_return_if_cxx_and_linker_flags_work(HAS_FLTO - "-flto${cxx_append}" "-flto${linker_append}" - PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) - endif() - elseif (CMAKE_CXX_COMPILER_ID MATCHES "Intel") - # Intel equivalent to LTO is called IPO - _pybind11_return_if_cxx_and_linker_flags_work(HAS_INTEL_IPO - "-ipo" "-ipo" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) - elseif(MSVC) - # cmake only interprets libraries as linker flags when they start with a - (otherwise it - # converts /LTCG to \LTCG as if it was a Windows path). Luckily MSVC supports passing flags - # with - instead of /, even if it is a bit non-standard: - _pybind11_return_if_cxx_and_linker_flags_work(HAS_MSVC_GL_LTCG - "/GL" "-LTCG" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) - endif() - - if (PYBIND11_LTO_CXX_FLAGS) - message(STATUS "LTO enabled") - else() - message(STATUS "LTO disabled (not supported by the compiler and/or linker)") - endif() - endif() - - # Enable LTO flags if found, except for Debug builds - if (PYBIND11_LTO_CXX_FLAGS) - target_compile_options(${target_name} PRIVATE "$<$>:${PYBIND11_LTO_CXX_FLAGS}>") - endif() - if (PYBIND11_LTO_LINKER_FLAGS) - target_link_libraries(${target_name} PRIVATE "$<$>:${PYBIND11_LTO_LINKER_FLAGS}>") - endif() -endfunction() - -# Build a Python extension module: -# pybind11_add_module( [MODULE | SHARED] [EXCLUDE_FROM_ALL] -# [NO_EXTRAS] [SYSTEM] [THIN_LTO] source1 [source2 ...]) -# -function(pybind11_add_module target_name) - set(options MODULE SHARED EXCLUDE_FROM_ALL NO_EXTRAS SYSTEM THIN_LTO) - cmake_parse_arguments(ARG "${options}" "" "" ${ARGN}) - - if(ARG_MODULE AND ARG_SHARED) - message(FATAL_ERROR "Can't be both MODULE and SHARED") - elseif(ARG_SHARED) - set(lib_type SHARED) - else() - set(lib_type MODULE) - endif() - - if(ARG_EXCLUDE_FROM_ALL) - set(exclude_from_all EXCLUDE_FROM_ALL) - endif() - - add_library(${target_name} ${lib_type} ${exclude_from_all} ${ARG_UNPARSED_ARGUMENTS}) - message("ADDING LIBRARY ${target_name}") - - if(ARG_SYSTEM) - set(inc_isystem SYSTEM) - endif() - - target_include_directories(${target_name} ${inc_isystem} - PRIVATE ${PYBIND11_INCLUDE_DIR} # from project CMakeLists.txt - PRIVATE ${pybind11_INCLUDE_DIR} # from pybind11Config - PRIVATE ${PYTHON_INCLUDE_DIRS}) - - # Python debug libraries expose slightly different objects - # https://docs.python.org/3.6/c-api/intro.html#debugging-builds - # https://stackoverflow.com/questions/39161202/how-to-work-around-missing-pymodule-create2-in-amd64-win-python35-d-lib - if(PYTHON_IS_DEBUG) - target_compile_definitions(${target_name} PRIVATE Py_DEBUG) - endif() - - # The prefix and extension are provided by FindPythonLibsNew.cmake - set_target_properties(${target_name} PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}") - set_target_properties(${target_name} PROPERTIES SUFFIX "${PYTHON_MODULE_EXTENSION}") - - # -fvisibility=hidden is required to allow multiple modules compiled against - # different pybind versions to work properly, and for some features (e.g. - # py::module_local). We force it on everything inside the `pybind11` - # namespace; also turning it on for a pybind module compilation here avoids - # potential warnings or issues from having mixed hidden/non-hidden types. - set_target_properties(${target_name} PROPERTIES CXX_VISIBILITY_PRESET "hidden") - set_target_properties(${target_name} PROPERTIES CUDA_VISIBILITY_PRESET "hidden") - - if(WIN32 OR CYGWIN) - # Link against the Python shared library on Windows - target_link_libraries(${target_name} PRIVATE ${PYTHON_LIBRARIES}) - elseif(APPLE) - # It's quite common to have multiple copies of the same Python version - # installed on one's system. E.g.: one copy from the OS and another copy - # that's statically linked into an application like Blender or Maya. - # If we link our plugin library against the OS Python here and import it - # into Blender or Maya later on, this will cause segfaults when multiple - # conflicting Python instances are active at the same time (even when they - # are of the same version). - - # Windows is not affected by this issue since it handles DLL imports - # differently. The solution for Linux and Mac OS is simple: we just don't - # link against the Python library. The resulting shared library will have - # missing symbols, but that's perfectly fine -- they will be resolved at - # import time. - - target_link_libraries(${target_name} PRIVATE "-undefined dynamic_lookup") - - if(ARG_SHARED) - # Suppress CMake >= 3.0 warning for shared libraries - set_target_properties(${target_name} PROPERTIES MACOSX_RPATH ON) - endif() - endif() - - # Make sure C++11/14 are enabled - target_compile_options(${target_name} PUBLIC ${PYBIND11_CPP_STANDARD}) - - if(ARG_NO_EXTRAS) - return() - endif() - - _pybind11_add_lto_flags(${target_name} ${ARG_THIN_LTO}) - - if (NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug) - # Strip unnecessary sections of the binary on Linux/Mac OS - if(CMAKE_STRIP) - if(APPLE) - add_custom_command(TARGET ${target_name} POST_BUILD - COMMAND ${CMAKE_STRIP} -x $) - else() - add_custom_command(TARGET ${target_name} POST_BUILD - COMMAND ${CMAKE_STRIP} $) - endif() - endif() - endif() - - if(MSVC) - # /MP enables multithreaded builds (relevant when there are many files), /bigobj is - # needed for bigger binding projects due to the limit to 64k addressable sections - target_compile_options(${target_name} PRIVATE /bigobj) - if(CMAKE_VERSION VERSION_LESS 3.11) - target_compile_options(${target_name} PRIVATE $<$>:/MP>) - else() - # Only set these options for C++ files. This is important so that, for - # instance, projects that include other types of source files like CUDA - # .cu files don't get these options propagated to nvcc since that would - # cause the build to fail. - target_compile_options(${target_name} PRIVATE $<$>:$<$:/MP>>) - endif() - endif() -endfunction() \ No newline at end of file diff --git a/torchaudio/csrc/decoder/src/decoder/lm/KenLM.cpp b/torchaudio/csrc/decoder/src/decoder/lm/KenLM.cpp index 6d7bc52834..591a27da01 100644 --- a/torchaudio/csrc/decoder/src/decoder/lm/KenLM.cpp +++ b/torchaudio/csrc/decoder/src/decoder/lm/KenLM.cpp @@ -9,7 +9,7 @@ #include -#include "kenlm/lm/model.hh" +#include "lm/model.hh" namespace torchaudio { namespace lib {