Skip to content

Commit

Permalink
Remove Protobuf load/export feature (#193)
Browse files Browse the repository at this point in the history
  • Loading branch information
hcho3 authored Aug 28, 2020
1 parent 70bdebe commit 996114f
Show file tree
Hide file tree
Showing 22 changed files with 26 additions and 793 deletions.
8 changes: 0 additions & 8 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,6 @@
*.gch
*.pch

# Protobuf generated files
*.pb.h
*.pb.cc
*_pb2.py
*_pb2.pyc
*_pb3.py
*_pb3.pyc

# Python compiled
*.pyc

Expand Down
7 changes: 1 addition & 6 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,7 @@ jobs:
env: TASK=cmake_import_test
# Test installation of Python source distribution
- os: linux
env: TASK=python_sdist_test USE_SYSTEM_PROTOBUF=yes
- os: linux
env: TASK=python_sdist_test USE_SYSTEM_PROTOBUF=no
# Test build with Protobuf disabled
- os: linux
env: TASK=build_without_protobuf
env: TASK=python_sdist_test
# Python package test
- os: osx
env: TASK=python_test
Expand Down
1 change: 0 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ if(MSVC)
endif()
endif()

option(ENABLE_PROTOBUF "Enable Protobuf" OFF)
option(TEST_COVERAGE "C++ test coverage" OFF)
option(USE_OPENMP "Use OpenMP" ON)
option(BUILD_CPP_TEST "Build C++ tests" OFF)
Expand Down
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ doxygen:
cd docs; doxygen

cpp-coverage:
rm -rf build; mkdir build; cd build; cmake .. -DTEST_COVERAGE=ON -DENABLE_PROTOBUF=ON -DCMAKE_BUILD_TYPE=Debug && make -j$(NPROC)
rm -rf build; mkdir build; cd build; cmake .. -DTEST_COVERAGE=ON -DCMAKE_BUILD_TYPE=Debug && make -j$(NPROC)

all:
rm -rf build; mkdir build; cd build; cmake .. -DENABLE_PROTOBUF=ON && make -j$(NPROC)
rm -rf build; mkdir build; cd build; cmake .. && make -j$(NPROC)

pippack:
cd python && python setup.py sdist && mv dist/*.tar.gz .. && cd ../runtime/python && python setup.py sdist && mv dist/*.tar.gz ../..
8 changes: 4 additions & 4 deletions azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ jobs:
vmImage: 'ubuntu-latest'
steps:
- checkout: self
- script: tests/ci_build/ci_build.sh cpu tests/ci_build/build_via_cmake.sh -DENABLE_PROTOBUF=ON
- script: tests/ci_build/ci_build.sh cpu tests/ci_build/build_via_cmake.sh
displayName: 'Building Treelite...'
- script: |
tests/ci_build/ci_build.sh cpu bash -c "cd python/ && python setup.py bdist_wheel --universal"
Expand Down Expand Up @@ -50,7 +50,7 @@ jobs:
- script: |
mkdir build
cd build
cmake .. -G"Visual Studio 15 2017 Win64" -DENABLE_PROTOBUF=ON -DBUILD_CPP_TEST=ON
cmake .. -G"Visual Studio 15 2017 Win64" -DBUILD_CPP_TEST=ON
displayName: 'Generating Visual Studio solution...'
- task: MSBuild@1
inputs:
Expand Down Expand Up @@ -93,7 +93,7 @@ jobs:
- script: |
mkdir build/
cd build/
cmake .. -DTEST_COVERAGE=ON -DENABLE_PROTOBUF=ON -DCMAKE_BUILD_TYPE=Debug -DBUILD_CPP_TEST=ON -GNinja
cmake .. -DTEST_COVERAGE=ON -DCMAKE_BUILD_TYPE=Debug -DBUILD_CPP_TEST=ON -GNinja
ninja
displayName: 'Building Treelite...'
- script: python -m pip install --upgrade pip numpy scipy pandas pytest pytest-cov scikit-learn xgboost lightgbm
Expand Down Expand Up @@ -131,7 +131,7 @@ jobs:
- script: |
mkdir build
cd build
cmake .. -G"Visual Studio 16 2019" -A x64 -DENABLE_PROTOBUF=ON
cmake .. -G"Visual Studio 16 2019" -A x64
displayName: 'Generating Visual Studio solution...'
- task: MSBuild@1
inputs:
Expand Down
23 changes: 0 additions & 23 deletions cmake/ExternalLibs.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -15,29 +15,6 @@ FetchContent_Declare(
FetchContent_MakeAvailable(fmtlib)
set_target_properties(fmt PROPERTIES EXCLUDE_FROM_ALL TRUE)

# Protobuf
if(ENABLE_PROTOBUF)
set(Protobuf_USE_STATIC_LIBS ON)
find_package(Protobuf)
if(NOT Protobuf_FOUND)
message(STATUS "Did not found Protobuf in the system root. Fetching Protobuf now...")
set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build tests for protobuf" FORCE)
set(protobuf_BUILD_SHARED_LIBS OFF CACHE BOOL "enable shared libs for protobuf" FORCE)
FetchContent_Populate(protobuf
GIT_REPOSITORY https://github.com/protocolbuffers/protobuf
GIT_TAG v3.12.3
SOURCE_SUBDIR cmake
)
add_subdirectory(${protobuf_SOURCE_DIR}/cmake ${protobuf_BINARY_DIR})
set(Protobuf_PROTOC_EXECUTABLE protobuf::protoc)
set(Protobuf_INCLUDE_DIRS ${protobuf_SOURCE_DIR}/src)
set(Protobuf_LIBRARIES protobuf::libprotobuf)
set_target_properties(libprotobuf PROPERTIES POSITION_INDEPENDENT_CODE ON)
endif()
else()
set(Protobuf_LIBRARIES "")
endif()

# Google C++ tests
if(BUILD_CPP_TEST)
find_package(GTest)
Expand Down
4 changes: 0 additions & 4 deletions cmake/TreeliteConfig.cmake.in
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,9 @@
include(CMakeFindDependencyMacro)

set(USE_OPENMP @USE_OPENMP@)
set(ENABLE_PROTOBUF @ENABLE_PROTOBUF@)
if(USE_OPENMP)
find_dependency(OpenMP)
endif()
if(ENABLE_PROTOBUF)
find_dependency(Protobuf)
endif()
if(NOT TARGET treelite::treelite)
include(${CMAKE_CURRENT_LIST_DIR}/TreeliteTargets.cmake)
endif()
Expand Down
125 changes: 0 additions & 125 deletions cmake/Utils.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -40,128 +40,3 @@ function(set_default_configuration_release)
set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE )
endif()
endfunction(set_default_configuration_release)

#=============================================================================
# Copyright 2009 Kitware, Inc.
# Copyright 2009-2011 Philip Lowman <[email protected]>
# Copyright 2008 Esben Mose Hansen, Ange Optimization ApS
#
# Distributed under the OSI-approved BSD License (the "License");
# see accompanying file Copyright.txt for details.
#
# This software is distributed WITHOUT ANY WARRANTY; without even the
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the License for more information.
#=============================================================================
# (To distribute this file outside of CMake, substitute the full
# License text for the above reference.)
function(PROTOBUF_GENERATE_CPP SRCS HDRS)
if(NOT ARGN)
message(SEND_ERROR "Error: PROTOBUF_GENERATE_CPP() called without any proto files")
return()
endif()

# Create an include path for each file specified
foreach(FIL ${ARGN})
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
get_filename_component(ABS_PATH ${ABS_FIL} PATH)
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
if(${_contains_already} EQUAL -1)
list(APPEND _protobuf_include_path -I ${ABS_PATH})
endif()
endforeach()

if(DEFINED Protobuf_IMPORT_DIRS)
foreach(DIR ${Protobuf_IMPORT_DIRS})
get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
if(${_contains_already} EQUAL -1)
list(APPEND _protobuf_include_path -I ${ABS_PATH})
endif()
endforeach()
endif()

set(PROTOC_DEPENDENCY ${Protobuf_PROTOC_EXECUTABLE})

set(${SRCS})
set(${HDRS})
foreach(FIL ${ARGN})
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
get_filename_component(FIL_WE ${FIL} NAME_WE)

list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")

add_custom_command(
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
COMMAND ${Protobuf_PROTOC_EXECUTABLE}
ARGS --cpp_out ${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL}
DEPENDS ${ABS_FIL} ${PROTOC_DEPENDENCY}
COMMENT "Running C++ protocol buffer compiler on ${FIL}"
VERBATIM
)
endforeach()

set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
set(${SRCS} ${${SRCS}} PARENT_SCOPE)
set(${HDRS} ${${HDRS}} PARENT_SCOPE)
endfunction()

function(PROTOBUF_GENERATE_JAVA TARGET_NAME PROTO_FILE)
get_filename_component(ABS_FIL ${PROTO_FILE} ABSOLUTE)
get_filename_component(ABS_PATH ${ABS_FIL} PATH)
get_filename_component(FIL_WE ${PROTO_FILE} NAME_WE)
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
if(${_contains_already} EQUAL -1)
list(APPEND _protobuf_include_path -I ${ABS_PATH})
endif()

if(DEFINED Protobuf_IMPORT_DIRS)
foreach(DIR ${Protobuf_IMPORT_DIRS})
get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
if(${_contains_already} EQUAL -1)
list(APPEND _protobuf_include_path -I ${ABS_PATH})
endif()
endforeach()
endif()

set(PROTOC_DEPENDENCY ${Protobuf_PROTOC_EXECUTABLE})

add_custom_target(${TARGET_NAME} ALL
${Protobuf_PROTOC_EXECUTABLE} --java_out ${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL}
DEPENDS ${ABS_FIL} ${PROTOC_DEPENDENCY}
COMMENT "Running Java protocol buffer compiler on ${PROTO_FILE}"
VERBATIM
)
endfunction()

function(PROTOBUF_GENERATE_PYTHON TARGET_NAME PROTO_FILE)
get_filename_component(ABS_FIL ${PROTO_FILE} ABSOLUTE)
get_filename_component(ABS_PATH ${ABS_FIL} PATH)
get_filename_component(FIL_WE ${PROTO_FILE} NAME_WE)
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
if(${_contains_already} EQUAL -1)
list(APPEND _protobuf_include_path -I ${ABS_PATH})
endif()

if(DEFINED Protobuf_IMPORT_DIRS)
foreach(DIR ${Protobuf_IMPORT_DIRS})
get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
if(${_contains_already} EQUAL -1)
list(APPEND _protobuf_include_path -I ${ABS_PATH})
endif()
endforeach()
endif()

set(PROTOC_DEPENDENCY ${Protobuf_PROTOC_EXECUTABLE})

add_custom_target(${TARGET_NAME} ALL
${Protobuf_PROTOC_EXECUTABLE} --python_out ${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL}
DEPENDS ${ABS_FIL} ${PROTOC_DEPENDENCY}
COMMENT "Running Python protocol buffer compiler on ${PROTO_FILE}"
VERBATIM
)
endfunction()
3 changes: 1 addition & 2 deletions docs/tutorials/import.rst
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,7 @@ To import scikit-learn models, use
How about other packages?
-------------------------
If you used other packages to train your ensemble model, you'd need to specify
the model programmatically. There are two ways to do this:
the model programmatically:

* :doc:`builder`
* :doc:`protobuf`

1 change: 0 additions & 1 deletion docs/tutorials/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,3 @@ This page lists tutorials about Treelite.
deploy
deploy_java
builder
protobuf
34 changes: 0 additions & 34 deletions docs/tutorials/protobuf.rst

This file was deleted.

20 changes: 0 additions & 20 deletions include/treelite/c_api.h
Original file line number Diff line number Diff line change
Expand Up @@ -250,26 +250,6 @@ TREELITE_DLL int TreeliteLoadXGBoostModel(const char* filename,
TREELITE_DLL int TreeliteLoadXGBoostModelFromMemoryBuffer(const void* buf,
size_t len,
ModelHandle* out);
/*!
* \brief load a model in Protocol Buffers format. Protocol Buffers
* (google/protobuf) is a language- and platform-neutral mechanism for
* serializing structured data. See tree.proto for format spec.
* \param filename name of model file
* \param out loaded model
* \return 0 for success, -1 for failure
*/
TREELITE_DLL int TreeliteLoadProtobufModel(const char* filename,
ModelHandle* out);
/*!
* \brief export a model in Protocol Buffers format. Protocol Buffers
* (google/protobuf) is a language- and platform-neutral mechanism for
* serializing structured data. See src/tree.proto for format spec.
* \param filename name of model file
* \param model model to export
* \return 0 for success, -1 for failure
*/
TREELITE_DLL int TreeliteExportProtobufModel(const char* filename,
ModelHandle model);
/*!
* \brief Query the number of trees in the model
* \param handle model to query
Expand Down
16 changes: 0 additions & 16 deletions include/treelite/frontend.h
Original file line number Diff line number Diff line change
Expand Up @@ -42,22 +42,6 @@ void LoadXGBoostModel(const char* filename, Model* out);
* \param out reference to loaded model
*/
void LoadXGBoostModel(const void* buf, size_t len, Model* out);
/*!
* \brief load a model in Protocol Buffers format. Protocol Buffers
* (google/protobuf) is a language- and platform-neutral mechanism for
* serializing structured data. See tree.proto for format spec.
* \param filename name of model file
* \param out reference to loaded model
*/
void LoadProtobufModel(const char* filename, Model* out);
/*!
* \brief export a model in Protocol Buffers format. Protocol Buffers
* (google/protobuf) is a language- and platform-neutral mechanism for
* serializing structured data. See src/tree.proto for format spec.
* \param filename name of model file
* \param model model to export
*/
void ExportProtobufModel(const char* filename, const Model& model);

//--------------------------------------------------------------------------
// model builder interface: build trees incrementally
Expand Down
1 change: 0 additions & 1 deletion python/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,6 @@ class BuildExt(build_ext.build_ext): # pylint: disable=too-many-ancestors
def build(self, src_dir, build_dir, generator, build_tool=None):
"""Build the core library with CMake."""
cmake_cmd = ['cmake', src_dir, generator]
cmake_cmd.append('-DENABLE_PROTOBUF=ON')

self.logger.info('Run CMake command: %s', str(cmake_cmd))
subprocess.check_call(cmake_cmd, cwd=build_dir)
Expand Down
Loading

0 comments on commit 996114f

Please sign in to comment.