Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make HDF5 components optional for configurable build #2619

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,19 @@ set(CAFFE_TARGET_VERSION "1.0.0-rc3" CACHE STRING "Caffe logical version")
set(CAFFE_TARGET_SOVERSION "1.0.0-rc3" CACHE STRING "Caffe soname version")
add_definitions(-DCAFFE_VERSION=${CAFFE_TARGET_VERSION})

# Search packages for host system instead of packages for target system
# in case of cross compilation these macro should be defined by toolchain file
if(NOT COMMAND find_host_package)
macro(find_host_package)
find_package(${ARGN})
endmacro()
endif()
if(NOT COMMAND find_host_program)
macro(find_host_program)
find_program(${ARGN})
endmacro()
endif()

# ---[ Using cmake scripts and modules
list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules)

Expand All @@ -38,6 +51,7 @@ caffe_option(USE_OPENCV "Build with OpenCV support" ON)
caffe_option(USE_LEVELDB "Build with levelDB" ON)
caffe_option(USE_LMDB "Build with lmdb" ON)
caffe_option(ALLOW_LMDB_NOLOCK "Allow MDB_NOLOCK when reading LMDB files (only if necessary)" OFF)
caffe_option(USE_HDF5 "Build with hdf5" ON)

# ---[ Dependencies
include(cmake/Dependencies.cmake)
Expand Down
13 changes: 10 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -178,11 +178,12 @@ ifneq ($(CPU_ONLY), 1)
LIBRARIES := cudart cublas curand
endif

LIBRARIES += glog gflags protobuf boost_system boost_filesystem m hdf5_hl hdf5
LIBRARIES += glog gflags protobuf boost_system boost_filesystem m

# handle IO dependencies
USE_LEVELDB ?= 1
USE_LMDB ?= 1
USE_HDF5 ?= 1
USE_OPENCV ?= 1

ifeq ($(USE_LEVELDB), 1)
Expand All @@ -191,13 +192,16 @@ endif
ifeq ($(USE_LMDB), 1)
LIBRARIES += lmdb
endif
ifeq ($(USE_HDF5), 1)
LIBRARIES += hdf5_hl hdf5
endif
ifeq ($(USE_OPENCV), 1)
LIBRARIES += opencv_core opencv_highgui opencv_imgproc
LIBRARIES += opencv_core opencv_highgui opencv_imgproc

ifeq ($(OPENCV_VERSION), 3)
LIBRARIES += opencv_imgcodecs
endif

endif
PYTHON_LIBRARIES ?= boost_python python2.7
WARNINGS := -Wall -Wno-sign-compare
Expand Down Expand Up @@ -341,6 +345,9 @@ ifeq ($(ALLOW_LMDB_NOLOCK), 1)
COMMON_FLAGS += -DALLOW_LMDB_NOLOCK
endif
endif
ifeq ($(USE_HDF5), 1)
COMMON_FLAGS += -DUSE_HDF5
endif

# CPU-only configuration
ifeq ($(CPU_ONLY), 1)
Expand Down
1 change: 1 addition & 0 deletions Makefile.config.example
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
# USE_OPENCV := 0
# USE_LEVELDB := 0
# USE_LMDB := 0
# USE_HDF5 := 0

# uncomment to allow MDB_NOLOCK when reading LMDB files (only if necessary)
# You should not set this flag if you will be reading LMDBs with any
Expand Down
4 changes: 4 additions & 0 deletions cmake/ConfigGen.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,10 @@ function(caffe_generate_export_configs)
list(APPEND Caffe_DEFINITIONS -DUSE_LEVELDB)
endif()

if(USE_HDF5)
list(APPEND Caffe_DEFINITIONS -DUSE_HDF5)
endif()

if(NOT HAVE_CUDNN)
set(HAVE_CUDNN FALSE)
else()
Expand Down
17 changes: 10 additions & 7 deletions cmake/Dependencies.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,12 @@ list(APPEND Caffe_LINKER_LIBS ${GFLAGS_LIBRARIES})
include(cmake/ProtoBuf.cmake)

# ---[ HDF5
find_package(HDF5 COMPONENTS HL REQUIRED)
include_directories(SYSTEM ${HDF5_INCLUDE_DIRS} ${HDF5_HL_INCLUDE_DIR})
list(APPEND Caffe_LINKER_LIBS ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES})
if(USE_HDF5)
find_package(HDF5 COMPONENTS HL REQUIRED)
include_directories(SYSTEM ${HDF5_INCLUDE_DIRS} ${HDF5_HL_INCLUDE_DIR})
list(APPEND Caffe_LINKER_LIBS ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES})
add_definitions(-DUSE_HDF5)
endif()

# ---[ LMDB
if(USE_LMDB)
Expand Down Expand Up @@ -113,18 +116,18 @@ if(BUILD_python)
find_package(NumPy 1.7.1)
# Find the matching boost python implementation
set(version ${PYTHONLIBS_VERSION_STRING})

STRING( REGEX REPLACE "[^0-9]" "" boost_py_version ${version} )
find_package(Boost 1.46 COMPONENTS "python-py${boost_py_version}")
set(Boost_PYTHON_FOUND ${Boost_PYTHON-PY${boost_py_version}_FOUND})

while(NOT "${version}" STREQUAL "" AND NOT Boost_PYTHON_FOUND)
STRING( REGEX REPLACE "([0-9.]+).[0-9]+" "\\1" version ${version} )

STRING( REGEX REPLACE "[^0-9]" "" boost_py_version ${version} )
find_package(Boost 1.46 COMPONENTS "python-py${boost_py_version}")
set(Boost_PYTHON_FOUND ${Boost_PYTHON-PY${boost_py_version}_FOUND})

STRING( REGEX MATCHALL "([0-9.]+).[0-9]+" has_more_version ${version} )
if("${has_more_version}" STREQUAL "")
break()
Expand Down
231 changes: 231 additions & 0 deletions cmake/Modules/FindProtobuf.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,231 @@
# Locate and configure the Google Protocol Buffers library.
#
# The following variables can be set and are optional:
#
# PROTOBUF_SRC_ROOT_FOLDER - When compiling with MSVC, if this cache variable is set
# the protobuf-default VS project build locations
# (vsprojects/Debug & vsprojects/Release) will be searched
# for libraries and binaries.
#
# PROTOBUF_IMPORT_DIRS - List of additional directories to be searched for
# imported .proto files. (New in CMake 2.8.8)
#
# Defines the following variables:
#
# PROTOBUF_FOUND - Found the Google Protocol Buffers library (libprotobuf & header files)
# PROTOBUF_INCLUDE_DIRS - Include directories for Google Protocol Buffers
# PROTOBUF_LIBRARIES - The protobuf libraries
# [New in CMake 2.8.5]
# PROTOBUF_PROTOC_LIBRARIES - The protoc libraries
# PROTOBUF_LITE_LIBRARIES - The protobuf-lite libraries
#
# The following cache variables are also available to set or use:
# PROTOBUF_LIBRARY - The protobuf library
# PROTOBUF_PROTOC_LIBRARY - The protoc library
# PROTOBUF_INCLUDE_DIR - The include directory for protocol buffers
# PROTOBUF_PROTOC_EXECUTABLE - The protoc compiler
# [New in CMake 2.8.5]
# PROTOBUF_LIBRARY_DEBUG - The protobuf library (debug)
# PROTOBUF_PROTOC_LIBRARY_DEBUG - The protoc library (debug)
# PROTOBUF_LITE_LIBRARY - The protobuf lite library
# PROTOBUF_LITE_LIBRARY_DEBUG - The protobuf lite library (debug)
#
# ====================================================================
# Example:
#
# find_package(Protobuf REQUIRED)
# include_directories(${PROTOBUF_INCLUDE_DIRS})
#
# include_directories(${CMAKE_CURRENT_BINARY_DIR})
# PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS foo.proto)
# add_executable(bar bar.cc ${PROTO_SRCS} ${PROTO_HDRS})
# target_link_libraries(bar ${PROTOBUF_LIBRARIES})
#
# NOTE: You may need to link against pthreads, depending
# on the platform.
#
# NOTE: The PROTOBUF_GENERATE_CPP macro & add_executable() or add_library()
# calls only work properly within the same directory.
#
# ====================================================================
#
# PROTOBUF_GENERATE_CPP (public function)
# SRCS = Variable to define with autogenerated
# source files
# HDRS = Variable to define with autogenerated
# header files
# ARGN = proto files
#
# ====================================================================


#=============================================================================
# Copyright 2009 Kitware, Inc.
# Copyright 2009-2011 Philip Lowman <philip@yhbt.com>
# Copyright 2008 Esben Mose Hansen, Ange Optimization ApS
#
# Distributed under the OSI-approved BSD License (the "License");
# see accompanying file Copyright.txt for details.
#
# This software is distributed WITHOUT ANY WARRANTY; without even the
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the License for more information.
#=============================================================================
# (To distribute this file outside of CMake, substitute the full
# License text for the above reference.)

function(PROTOBUF_GENERATE_CPP SRCS HDRS)
if(NOT ARGN)
message(SEND_ERROR "Error: PROTOBUF_GENERATE_CPP() called without any proto files")
return()
endif()

if(PROTOBUF_GENERATE_CPP_APPEND_PATH)
# Create an include path for each file specified
foreach(FIL ${ARGN})
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
get_filename_component(ABS_PATH ${ABS_FIL} PATH)
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
if(${_contains_already} EQUAL -1)
list(APPEND _protobuf_include_path -I ${ABS_PATH})
endif()
endforeach()
else()
set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})
endif()

if(DEFINED PROTOBUF_IMPORT_DIRS)
foreach(DIR ${PROTOBUF_IMPORT_DIRS})
get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
if(${_contains_already} EQUAL -1)
list(APPEND _protobuf_include_path -I ${ABS_PATH})
endif()
endforeach()
endif()

set(${SRCS})
set(${HDRS})
foreach(FIL ${ARGN})
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
get_filename_component(FIL_WE ${FIL} NAME_WE)

list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")

add_custom_command(
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
COMMAND ${PROTOBUF_PROTOC_EXECUTABLE}
ARGS --cpp_out ${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL}
DEPENDS ${ABS_FIL}
COMMENT "Running C++ protocol buffer compiler on ${FIL}"
VERBATIM )
endforeach()

set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
set(${SRCS} ${${SRCS}} PARENT_SCOPE)
set(${HDRS} ${${HDRS}} PARENT_SCOPE)
endfunction()

# Internal function: search for normal library as well as a debug one
# if the debug one is specified also include debug/optimized keywords
# in *_LIBRARIES variable
function(_protobuf_find_libraries name filename)
find_library(${name}_LIBRARY
NAMES ${filename}
PATHS ${PROTOBUF_SRC_ROOT_FOLDER}/vsprojects/Release)
mark_as_advanced(${name}_LIBRARY)

find_library(${name}_LIBRARY_DEBUG
NAMES ${filename}
PATHS ${PROTOBUF_SRC_ROOT_FOLDER}/vsprojects/Debug)
mark_as_advanced(${name}_LIBRARY_DEBUG)

if(NOT ${name}_LIBRARY_DEBUG)
# There is no debug library
set(${name}_LIBRARY_DEBUG ${${name}_LIBRARY} PARENT_SCOPE)
set(${name}_LIBRARIES ${${name}_LIBRARY} PARENT_SCOPE)
else()
# There IS a debug library
set(${name}_LIBRARIES
optimized ${${name}_LIBRARY}
debug ${${name}_LIBRARY_DEBUG}
PARENT_SCOPE
)
endif()
endfunction()

# Internal function: find threads library
function(_protobuf_find_threads)
set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
find_package(Threads)
if(Threads_FOUND)
list(APPEND PROTOBUF_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})
set(PROTOBUF_LIBRARIES "${PROTOBUF_LIBRARIES}" PARENT_SCOPE)
endif()
endfunction()

#
# Main.
#

# By default have PROTOBUF_GENERATE_CPP macro pass -I to protoc
# for each directory where a proto file is referenced.
if(NOT DEFINED PROTOBUF_GENERATE_CPP_APPEND_PATH)
set(PROTOBUF_GENERATE_CPP_APPEND_PATH TRUE)
endif()


# Google's provided vcproj files generate libraries with a "lib"
# prefix on Windows
if(MSVC)
set(PROTOBUF_ORIG_FIND_LIBRARY_PREFIXES "${CMAKE_FIND_LIBRARY_PREFIXES}")
set(CMAKE_FIND_LIBRARY_PREFIXES "lib" "")

find_path(PROTOBUF_SRC_ROOT_FOLDER protobuf.pc.in)
endif()

# The Protobuf library
_protobuf_find_libraries(PROTOBUF protobuf)
#DOC "The Google Protocol Buffers RELEASE Library"

_protobuf_find_libraries(PROTOBUF_LITE protobuf-lite)

# The Protobuf Protoc Library
_protobuf_find_libraries(PROTOBUF_PROTOC protoc)

# Restore original find library prefixes
if(MSVC)
set(CMAKE_FIND_LIBRARY_PREFIXES "${PROTOBUF_ORIG_FIND_LIBRARY_PREFIXES}")
endif()

if(UNIX)
_protobuf_find_threads()
endif()

# Find the include directory
find_path(PROTOBUF_INCLUDE_DIR
google/protobuf/service.h
PATHS ${PROTOBUF_SRC_ROOT_FOLDER}/src
)
mark_as_advanced(PROTOBUF_INCLUDE_DIR)

# Find the protoc Executable
find_host_program(PROTOBUF_PROTOC_EXECUTABLE
NAMES protoc
DOC "The Google Protocol Buffers Compiler"
PATHS
${PROTOBUF_SRC_ROOT_FOLDER}/vsprojects/Release
${PROTOBUF_SRC_ROOT_FOLDER}/vsprojects/Debug
)
mark_as_advanced(PROTOBUF_PROTOC_EXECUTABLE)


include(FindPackageHandleStandardArgs)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(PROTOBUF DEFAULT_MSG
PROTOBUF_LIBRARY PROTOBUF_INCLUDE_DIR)

if(PROTOBUF_FOUND)
set(PROTOBUF_INCLUDE_DIRS ${PROTOBUF_INCLUDE_DIR})
endif()
1 change: 1 addition & 0 deletions cmake/Summary.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ function(caffe_print_configuration_summary)
caffe_status(" USE_LEVELDB : ${USE_LEVELDB}")
caffe_status(" USE_LMDB : ${USE_LMDB}")
caffe_status(" ALLOW_LMDB_NOLOCK : ${ALLOW_LMDB_NOLOCK}")
caffe_status(" USE_HDF5 : ${USE_HDF5}")
caffe_status("")
caffe_status("Dependencies:")
caffe_status(" BLAS : " APPLE THEN "Yes (vecLib)" ELSE "Yes (${BLAS})")
Expand Down
1 change: 1 addition & 0 deletions cmake/Templates/caffe_config.h.in
Original file line number Diff line number Diff line change
Expand Up @@ -36,3 +36,4 @@
#cmakedefine USE_LEVELDB
#cmakedefine USE_LMDB
#cmakedefine ALLOW_LMDB_NOLOCK
#cmakedefine USE_HDF5
4 changes: 2 additions & 2 deletions docs/installation.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,12 @@ Caffe has several dependencies:
* 5.5, and 5.0 are compatible but considered legacy
* [BLAS](http://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) via ATLAS, MKL, or OpenBLAS.
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Making this optional too to get GPU only version?

* [Boost](http://www.boost.org/) >= 1.55
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(optional)

* `protobuf`, `glog`, `gflags`, `hdf5`
* `protobuf`, `glog`, `gflags`

Optional dependencies:

* [OpenCV](http://opencv.org/) >= 2.4 including 3.0
* IO libraries: `lmdb`, `leveldb` (note: leveldb requires `snappy`)
* IO libraries: `lmdb`, `leveldb` (note: leveldb requires `snappy`), `hdf5`
* cuDNN for GPU acceleration (v5)

Pycaffe and Matcaffe interfaces have their own natural needs.
Expand Down
2 changes: 2 additions & 0 deletions include/caffe/util/hdf5.hpp
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#ifdef USE_HDF5
#ifndef CAFFE_UTIL_HDF5_H_
#define CAFFE_UTIL_HDF5_H_

Expand Down Expand Up @@ -37,3 +38,4 @@ string hdf5_get_name_by_idx(hid_t loc_id, int idx);
} // namespace caffe

#endif // CAFFE_UTIL_HDF5_H_
#endif // USE_HDF5
Loading