Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[coreml-tools] Update to 8.0, detect Python3 version, site-packages folder #287

Merged
merged 3 commits into from
Oct 31, 2024
Merged

[coreml-tools] Update to 8.0, detect Python3 version, site-packages folder #287

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c5f4b41
[coreml-tools] writing python helpers
luncliff Oct 29, 2024
1e8aa24
[coreml-tools] update portfile,patches
luncliff Oct 30, 2024
7e68207
[coreml-tools] update to 8.0
luncliff Oct 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
227 changes: 70 additions & 157 deletions ports/coreml-tools/fix-cmake.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,75 +1,59 @@
diff --git a/CMakeLists.txt b/CMakeLists.txt
index d462525..1f97838 100644
index d462525..315a9ee 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -3,6 +3,7 @@ cmake_minimum_required(VERSION 3.10.2)
set(CMAKE_DISABLE_IN_SOURCE_BUILD ON)

project(coremltools)
+include(GNUInstallDirs)

if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}")
message(FATAL_ERROR "
@@ -31,34 +32,46 @@ if(HAS_CCACHE)
@@ -31,7 +31,14 @@ if(HAS_CCACHE)
set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
endif()

-add_subdirectory(deps)
-add_subdirectory(mlmodel)
+find_package(Python3 COMPONENTS Interpreter REQUIRED)
+if(NOT CMAKE_CROSSCOMPILING)
+ find_package(Python3 COMPONENTS Development NumPy REQUIRED)
+ link_libraries(Python3::Python Python3::NumPy)
+endif()
+
+find_package(pybind11 CONFIG)
+if(pybind11_FOUND)
+ include_directories(${pybind11_INCLUDE_DIRS})
+ link_libraries(${pybind11_LIBRARIES})
+ add_compile_definitions(${pybind11_DEFINITIONS})
+endif()
+find_package(nlohmann_json CONFIG REQUIRED)
+find_package(protobuf CONFIG REQUIRED)
add_subdirectory(mlmodel)

-find_package(PythonInterp)
-find_package(PythonLibs)
+find_package(nlohmann_json CONFIG REQUIRED) # nlohmann_json::nlohmann_json
+link_libraries(nlohmann_json::nlohmann_json)

-message("Found python at ${PYTHON_EXECUTABLE}")
-message("Found python version ${PYTHON_VERSION_STRING}")
-message("Found python includes ${PYTHON_INCLUDE_DIRS}")
+find_package(protobuf CONFIG REQUIRED) # protobuf::libprotobuf
+link_libraries("-framework Foundation")
+
+add_subdirectory(mlmodel)
find_package(PythonInterp)
@@ -41,17 +48,23 @@ message("Found python at ${PYTHON_EXECUTABLE}")
message("Found python version ${PYTHON_VERSION_STRING}")
message("Found python includes ${PYTHON_INCLUDE_DIRS}")

+link_libraries(
+ protobuf::libprotobuf
+ nlohmann_json::nlohmann_json
+)
include_directories(
.
- deps/protobuf/src
- deps/pybind11/include
- deps/nlohmann
mlmodel/src
- ${PYTHON_INCLUDE_DIRS}
+ ${pybind11_INCLUDE_DIRS}
mlmodel/src
${PYTHON_INCLUDE_DIRS}
)

if(APPLE)
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fobjc-arc")
+ add_compile_options(-fobjc-arc)
endif()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fobjc-arc")
+ link_libraries("-framework Foundation")
+endif()
+if(CMAKE_CROSSCOMPILING)
+ return()
+endif() # just libraries. no executables
+ return() # just libraries. no executables
endif()

add_library(milstoragepython
SHARED
@@ -59,6 +72,7 @@ add_library(milstoragepython
milstoragepython/MilStorage.cpp
milstoragepython/MilStoragePython.cpp
)
+install(TARGETS milstoragepython DESTINATION ${CMAKE_INSTALL_LIBDIR})

target_link_libraries(milstoragepython
mlmodel
@@ -70,7 +86,8 @@ add_library(modelpackage
@@ -70,7 +84,8 @@ add_library(modelpackage
modelpackage/src/utils/JsonMap.cpp
modelpackage/src/ModelPackagePython.cpp
)
Expand All @@ -79,39 +63,15 @@ index d462525..1f97838 100644
target_compile_definitions(modelpackage
PRIVATE
CPU_ONLY=1
@@ -78,7 +95,7 @@ target_compile_definitions(modelpackage
@@ -78,7 +93,6 @@ target_compile_definitions(modelpackage

target_link_libraries(modelpackage
mlmodel
- libprotobuf
+ protobuf::libprotobuf
)

if (CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.1)
@@ -132,21 +149,7 @@ find_library(CORE_VIDEO CoreVideo)
find_library(CORE_ML CoreML)
find_library(FOUNDATION Foundation)

-if (APPLE AND CORE_VIDEO AND CORE_ML AND FOUNDATION)
- execute_process(
- COMMAND ${PYTHON_EXECUTABLE} -c "import numpy; print(numpy.get_include())"
- RESULT_VARIABLE NUMPY_INCLUDE_STATUS
- OUTPUT_VARIABLE NUMPY_INCLUDE
- )
-
- if("${NUMPY_INCLUDE}" STREQUAL "" OR NOT NUMPY_INCLUDE_STATUS EQUAL 0)
- message(FATAL_ERROR "Could not find numpy include path. Exit code: ${NUMPY_INCLUDE_STATUS}")
- endif()
- message("Found numpy include path at ${NUMPY_INCLUDE}")
-
- include_directories(
- ${NUMPY_INCLUDE}
- )
+if (APPLE)

add_library(coremlpython
SHARED
@@ -157,6 +160,7 @@ if (APPLE AND CORE_VIDEO AND CORE_ML AND FOUNDATION)
@@ -157,6 +171,7 @@ if (APPLE AND CORE_VIDEO AND CORE_ML AND FOUNDATION)
coremlpython/CoreMLPythonUtils.mm
coremlpython/CoreMLPythonUtils.h
)
Expand All @@ -120,20 +80,19 @@ index d462525..1f97838 100644
mlmodel
${CORE_VIDEO}
diff --git a/cmake/coreml-utils.cmake b/cmake/coreml-utils.cmake
index ab4679c..259b4ef 100644
index ab4679c..e988cda 100644
--- a/cmake/coreml-utils.cmake
+++ b/cmake/coreml-utils.cmake
@@ -20,35 +20,55 @@
@@ -20,45 +20,43 @@
# to the list proto_depends in PARENT_SCOPE.
#
function(coreml_add_build_proto proto_fn target_suffix)
+ # some commands need mlmodel/enumgen host executable
+ if(CMAKE_CROSSCOMPILING)
+ find_program(ENUMGEN NAMES enumgen mlmodel/enumgen REQUIRED)
+ else()
+ set(ENUMGEN mlmodel${target_suffix}/enumgen)
+ set(ENUMGEN mlmodel/enumgen)
+ endif()
+
add_custom_command(
OUTPUT
${CMAKE_CURRENT_BINARY_DIR}/format/${proto_fn}.pb.cc
Expand All @@ -145,46 +104,23 @@ index ab4679c..259b4ef 100644
-I${CMAKE_CURRENT_SOURCE_DIR}/format
${CMAKE_CURRENT_SOURCE_DIR}/format/${proto_fn}.proto
- DEPENDS protoc
- WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
- )
- add_custom_command(
- OUTPUT
- ${CMAKE_CURRENT_BINARY_DIR}/format/${proto_fn}_enum.h
- COMMENT "Generating c++ enums from ${proto_fn}.proto into ${CMAKE_CURRENT_BINARY_DIR}/format/"
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
)
add_custom_command(
OUTPUT
${CMAKE_CURRENT_BINARY_DIR}/format/${proto_fn}_enum.h
COMMENT "Generating c++ enums from ${proto_fn}.proto into ${CMAKE_CURRENT_BINARY_DIR}/format/"
- COMMAND ${CMAKE_BINARY_DIR}/deps/protobuf/cmake/protoc
- --plugin=protoc-gen-enum=mlmodel/enumgen
- --enum_out=${CMAKE_CURRENT_BINARY_DIR}/format/
- -I${CMAKE_CURRENT_SOURCE_DIR}/format/
- ${CMAKE_CURRENT_SOURCE_DIR}/format/${proto_fn}.proto
+ COMMAND ${Protobuf_PROTOC_EXECUTABLE}
+ "--plugin=protoc-gen-enum=${ENUMGEN}"
--enum_out=${CMAKE_CURRENT_BINARY_DIR}/format/
-I${CMAKE_CURRENT_SOURCE_DIR}/format/
${CMAKE_CURRENT_SOURCE_DIR}/format/${proto_fn}.proto
- DEPENDS enumgen protoc
+ DEPENDS enumgen
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
)
+
+ if(NOT CMAKE_CROSSCOMPILING)
+ add_custom_command(
+ OUTPUT
+ ${CMAKE_CURRENT_BINARY_DIR}/format/${proto_fn}_enum.h
+ COMMENT "Generating c++ enums from ${proto_fn}.proto into ${CMAKE_CURRENT_BINARY_DIR}/format/"
+ COMMAND ${Protobuf_PROTOC_EXECUTABLE}
+ --plugin=protoc-gen-enum=${ENUMGEN}
+ --enum_out=${CMAKE_CURRENT_BINARY_DIR}/format/
+ -I${CMAKE_CURRENT_SOURCE_DIR}/format/
+ ${CMAKE_CURRENT_SOURCE_DIR}/format/${proto_fn}.proto
+ DEPENDS enumgen
+ WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+ )
+ else()
+ add_custom_command(
+ OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/format/${proto_fn}_enum.h
+ COMMAND ${Protobuf_PROTOC_EXECUTABLE}
+ --plugin=protoc-gen-enum=${ENUMGEN}
+ --enum_out=${CMAKE_CURRENT_BINARY_DIR}/format/
+ -I${CMAKE_CURRENT_SOURCE_DIR}/format/
+ ${CMAKE_CURRENT_SOURCE_DIR}/format/${proto_fn}.proto
+ WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+ )
+ endif()
+
add_custom_command(
OUTPUT
${CMAKE_BINARY_DIR}/coremltools${target_suffix}/proto/${proto_fn}_pb2.py
Expand All @@ -194,15 +130,17 @@ index ab4679c..259b4ef 100644
--python_out=${CMAKE_BINARY_DIR}/coremltools${target_suffix}/proto
-I${CMAKE_CURRENT_SOURCE_DIR}/format/
${CMAKE_CURRENT_SOURCE_DIR}/format/${proto_fn}.proto
@@ -58,7 +78,6 @@ function(coreml_add_build_proto proto_fn target_suffix)
--no-diff
-f import
${CMAKE_BINARY_DIR}/coremltools${target_suffix}/${proto_fn}_pb2.py
- COMMAND python
- -m lib2to3
- -wn
- --no-diff
- -f import
- ${CMAKE_BINARY_DIR}/coremltools${target_suffix}/${proto_fn}_pb2.py
- DEPENDS protoc
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
)
# For the CoreML framework we read the source file locations for these, and
@@ -67,26 +86,40 @@ function(coreml_add_build_proto proto_fn target_suffix)
@@ -67,36 +64,28 @@ function(coreml_add_build_proto proto_fn target_suffix)
if(OVERWRITE_PB_SOURCE)
add_custom_target(tgt_${proto_fn}_source ALL
COMMENT "Generating c++ sources from ${proto_fn}.proto into ${CMAKE_CURRENT_SOURCE_DIR}/build/format/"
Expand All @@ -212,84 +150,60 @@ index ab4679c..259b4ef 100644
-I${CMAKE_CURRENT_SOURCE_DIR}/format
${CMAKE_CURRENT_SOURCE_DIR}/format/${proto_fn}.proto
- DEPENDS protoc
- WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
- )
- add_custom_target(tgt_${proto_fn}_enums ALL
- COMMENT "Generating c++ enums from ${proto_fn}.proto into ${CMAKE_CURRENT_SOURCE_DIR}/build/format/"
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
)
add_custom_target(tgt_${proto_fn}_enums ALL
COMMENT "Generating c++ enums from ${proto_fn}.proto into ${CMAKE_CURRENT_SOURCE_DIR}/build/format/"
- COMMAND ${CMAKE_BINARY_DIR}/deps/protobuf/cmake/protoc
- --plugin=protoc-gen-enum=mlmodel/enumgen
- --enum_out=${CMAKE_CURRENT_SOURCE_DIR}/build/format/
- -I${CMAKE_CURRENT_SOURCE_DIR}/format/
- ${CMAKE_CURRENT_SOURCE_DIR}/format/${proto_fn}.proto
+ COMMAND ${Protobuf_PROTOC_EXECUTABLE}
+ "--plugin=protoc-gen-enum=${ENUMGEN}"
--enum_out=${CMAKE_CURRENT_SOURCE_DIR}/build/format/
-I${CMAKE_CURRENT_SOURCE_DIR}/format/
${CMAKE_CURRENT_SOURCE_DIR}/format/${proto_fn}.proto
- DEPENDS enumgen protoc
- WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+ WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+ DEPENDS enumgen
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
)
+
+ if(NOT CMAKE_CROSSCOMPILING)
+ add_custom_target(tgt_${proto_fn}_enums ALL
+ COMMENT "Generating c++ enums from ${proto_fn}.proto into ${CMAKE_CURRENT_SOURCE_DIR}/build/format/"
+ COMMAND ${Protobuf_PROTOC_EXECUTABLE}
+ --plugin=protoc-gen-enum=${ENUMGEN}
+ --enum_out=${CMAKE_CURRENT_SOURCE_DIR}/build/format/
+ -I${CMAKE_CURRENT_SOURCE_DIR}/format/
+ ${CMAKE_CURRENT_SOURCE_DIR}/format/${proto_fn}.proto
+ DEPENDS enumgen
+ WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+ )
+ else()
+ add_custom_target(tgt_${proto_fn}_enums ALL
+ COMMENT "Generating c++ enums from ${proto_fn}.proto into ${CMAKE_CURRENT_SOURCE_DIR}/build/format/"
+ COMMAND ${Protobuf_PROTOC_EXECUTABLE}
+ --plugin=protoc-gen-enum=${ENUMGEN}
+ --enum_out=${CMAKE_CURRENT_SOURCE_DIR}/build/format/
+ -I${CMAKE_CURRENT_SOURCE_DIR}/format/
+ ${CMAKE_CURRENT_SOURCE_DIR}/format/${proto_fn}.proto
+ DEPENDS enumgen
+ WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+ )
+ endif()
+
add_custom_target(tgt_${proto_fn}_python ALL
COMMENT "Generating Python sources from ${proto_fn}.proto into ${CMAKE_SOURCE_DIR}/coremltools${target_suffix}/proto/"
- COMMAND ${CMAKE_BINARY_DIR}/deps/protobuf/cmake/protoc
+ COMMAND ${Protobuf_PROTOC_EXECUTABLE}
--python_out=${CMAKE_SOURCE_DIR}/coremltools${target_suffix}/proto
-I${CMAKE_CURRENT_SOURCE_DIR}/format/
${CMAKE_CURRENT_SOURCE_DIR}/format/${proto_fn}.proto
@@ -96,8 +129,7 @@ function(coreml_add_build_proto proto_fn target_suffix)
--no-diff
-f import
${CMAKE_SOURCE_DIR}/coremltools${target_suffix}/proto/${proto_fn}_pb2.py
- COMMAND python
- -m lib2to3
- -wn
- --no-diff
- -f import
- ${CMAKE_SOURCE_DIR}/coremltools${target_suffix}/proto/${proto_fn}_pb2.py
- DEPENDS protoc
- WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+ WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
)
# Record dependencies for 'protosrc' target.
list(APPEND proto_depends tgt_${proto_fn}_source)
diff --git a/mlmodel/CMakeLists.txt b/mlmodel/CMakeLists.txt
index 6d7b7fa..85cf718 100644
index 14a57c0..fafbab7 100644
--- a/mlmodel/CMakeLists.txt
+++ b/mlmodel/CMakeLists.txt
@@ -2,8 +2,7 @@ include("${CMAKE_SOURCE_DIR}/cmake/coreml-utils.cmake")
@@ -2,8 +2,6 @@ include("${CMAKE_SOURCE_DIR}/cmake/coreml-utils.cmake")

include_directories(
..
- ../deps/FP16/include
- ../deps/protobuf/src
+# ../deps/FP16/include
src
)

@@ -64,6 +63,7 @@ target_link_libraries(mlmodel_test
@@ -64,6 +62,7 @@ target_link_libraries(mlmodel_test
add_executable(mlmodel_test_runner
test_runner.cpp
)
+install(TARGETS mlmodel_test_runner DESTINATION ${CMAKE_INSTALL_BINDIR})

target_link_libraries(mlmodel_test_runner
mlmodel_test
@@ -72,8 +72,6 @@ target_link_libraries(mlmodel_test_runner
@@ -72,8 +71,6 @@ target_link_libraries(mlmodel_test_runner
add_library(mlmodel
STATIC

Expand All @@ -298,12 +212,11 @@ index 6d7b7fa..85cf718 100644
${CMAKE_CURRENT_BINARY_DIR}/format/ArrayFeatureExtractor.pb.cc
${CMAKE_CURRENT_BINARY_DIR}/format/AudioFeaturePrint.pb.cc
${CMAKE_CURRENT_BINARY_DIR}/format/BayesianProbitRegressor.pb.cc
@@ -167,20 +165,22 @@ add_library(mlmodel
@@ -170,20 +167,21 @@ add_library(mlmodel
src/Validation/WordEmbeddingValidator.cpp
src/Validation/WordTaggerValidator.cpp
)
+install(TARGETS mlmodel DESTINATION ${CMAKE_INSTALL_LIBDIR})
+#

set_property(TARGET mlmodel
PROPERTY POSITION_INDEPENDENT_CODE ON
Expand All @@ -323,7 +236,7 @@ index 6d7b7fa..85cf718 100644

set(proto_files
ArrayFeatureExtractor
@@ -219,7 +218,7 @@ set(proto_files
@@ -222,7 +220,7 @@ set(proto_files
)

target_link_libraries(enumgen
Expand Down
Loading
Loading