Skip to content

Commit

Permalink
feat: Fixes, PDF, IIIF Auth, JPX compression parameters (#290)
Browse files Browse the repository at this point in the history 
* Bugfix for prepared statement with sqlite

* Added test for sqlite

* Newest libraries - serving pdfs...

* many Bugfixes

* Preparation for PDF support

* More work for PDF'as (PoDoFo integration)

* Added PDFWriter library for future treatment of PDF files

* Getting PDF's from J2K's

* PDF support

* Added page support for pdf's

PDF files can now be rendered with an extension of the IIIF syntax:
the image id may be extended by "@" and a pagenumber, e.g. "mypdf.pdf@3" will render only page 3 of the PDF file. In this case, region, size and rotation parameters do work and have to be supplied!

* IIIF Autentication

* bugfixes: now passes all tests!

* Changed command lline parser to CLI11 and added environment variables

* J2K parametrization

* feature (J2K compression parameters) Added to Lua

* feat: add JPX compression params

* test: Adding unit tests

testing for
1) PDF services
2) IIIF Auth API

* build: Add new kakadu library to travis

* build: Fix build inside docker (ongoing)

* build: Fix build inside docker (ongoing)

* wip: buiild problem

* build: Fix build inside docker (ongoing)

* chore: Add new minimum cmake version

* build: Fix poppler shared library requirements

* build: Turn debug off

* build: Fix podofo

* build: Fix fontconfig dependencies

* build: Fix fontconfig dependencies

* build: Use llvm under linux (ongoing)

* build: add separate clang compiler version check

* build: describe how to compile with gcc or clang

* build: add patch for bzip2 for compiling under mac

* build: everything static (ongoing)

* Build: problem solving half-way

* poppler inclusion

* Patch added

* build: add harfbuzz

* fix: Add missing header file

* build: all build and install artefacts are now inside the build directory

* build: fix for macOS

* build: fix for Linux

* build: update docker to new build directory structure

* build: add parameter to pytest for sipi executable
  • Loading branch information
@lrosenth @subotic
lrosenth authored and subotic committed Aug 29, 2019
1 parent 554d031 commit 6f46892
Show file tree
Hide file tree
Showing 104 changed files with 10,807 additions and 4,149 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ _build/
#
lib
local
build-linux
build-mac
cmake-build-debug
# ignore cache
cache/
Expand Down
6 changes: 3 additions & 3 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,14 @@ jobs:
- stage: test
script:
# decrypt archive containing the kakadu library
- openssl aes-256-cbc -K $encrypted_53935862068b_key -iv $encrypted_53935862068b_iv -in vendor/v7_A_5-01382N.zip.enc -out vendor/v7_A_5-01382N.zip -d
- openssl aes-256-cbc -K $encrypted_b6ef6f57f289_key -iv $encrypted_b6ef6f57f289_iv -in vendor/v7_A_7-01382N.zip.enc -out vendor/v7_A_7-01382N.zip -d
# run compile and tests inside docker
- docker run -v $PWD:/sipi dhlabbasel/sipi-base:18.04 /bin/sh -c "cd /sipi/build; cmake .. && make && ctest --verbose"
- docker run -it --rm -v $PWD:/sipi dhlabbasel/sipi-base:18.04 /bin/sh -c "mkdir -p /sipi/build-linux && cd /sipi/build-linux && cmake .. && make && ctest --verbose"

- stage: publish
script:
# decrypt archive containing the kakadu library
- openssl aes-256-cbc -K $encrypted_53935862068b_key -iv $encrypted_53935862068b_iv -in vendor/v7_A_5-01382N.zip.enc -out vendor/v7_A_5-01382N.zip -d
- openssl aes-256-cbc -K $encrypted_b6ef6f57f289_key -iv $encrypted_b6ef6f57f289_iv -in vendor/v7_A_7-01382N.zip.enc -out vendor/v7_A_7-01382N.zip -d
# login into docker hub, build image, and push but only when on the "develop" branch or tagged
# when on 'develop' then $TRAVIS_BRANCH = develop
# when on 'tag' then $TRAVIS_BRANCH = tag, e.g., v1.2.0
Expand Down
196 changes: 133 additions & 63 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
# License along with Sipi. If not, see <http://www.gnu.org/licenses/>.
#

cmake_minimum_required(VERSION 3.0.0)
cmake_minimum_required(VERSION 3.14.5)

include(ExternalProject)
include(CheckStructHasMember)
Expand All @@ -32,6 +32,29 @@ project(sipi)
string(TIMESTAMP SIPIDATE "%Y-%m-%d %H:%M")
set(SIPI_RELEASE "Sipi Version v1.5.0-SNAPSHOT (build ${SIPIDATE})")

# ------------------------------------------------------------------------------
# Set our path variables
# the build folder will hold all downloaded, built, and installed files
# ------------------------------------------------------------------------------
set(COMMON_LOCAL ${CMAKE_BINARY_DIR}/local)
set(COMMON_SRCS ${CMAKE_BINARY_DIR}/extsrcs)
set(CONFIGURE_LIBDIR ${COMMON_LOCAL}/lib)
set(CONFIGURE_INCDIR ${COMMON_LOCAL}/include)

# set cmake default install location
if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
set (CMAKE_INSTALL_PREFIX "${COMMON_LOCAL}" CACHE PATH "default install path" FORCE )
message(STATUS CMAKE_INSTALL_DIR: ${CMAKE_INSTALL_PREFIX})
endif()

set(COMMON_SOURCE_FILES_DIR ${PROJECT_SOURCE_DIR}/src)
set(COMMON_INCLUDE_FILES_DIR ${PROJECT_SOURCE_DIR}/include)
set(DOXYGEN_FILES_DIR ${PROJECT_SOURCE_DIR}/doxygen)
set(EXT_PROJECTS_DIR ${PROJECT_SOURCE_DIR}/ext)
set(COMMON_PATCHES ${PROJECT_SOURCE_DIR}/patches)
set(COMMON_VENDOR ${PROJECT_SOURCE_DIR}/vendor)
set(SIPI_SOURCE_DIR ${PROJECT_SOURCE_DIR})

#
# Usually SIPI is compiled with all static libraries
#
Expand All @@ -41,20 +64,36 @@ option(MAKE_DEBUG "Create sipi with debugger options on" ON)
#
# Here we determine the compiler and compiler version. We need clang >= 7.3 or g++ >= 5.3
#
if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "7.3")
message(FATAL_ERROR "Requires clang 7.3 or greater.")
if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "10.0")
message(FATAL_ERROR "Requires Apple Clang 10.0 or greater.")
set(ENV(CXX) "clang++")
set(ENV(CC) "clang")
endif()
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "7.0.0")
message(FATAL_ERROR "Requires Clang 7.0.0 or greater.")
set(ENV(CXX) "clang++")
set(ENV(CC) "clang")
endif()
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "5.3")
message(FATAL_ERROR "Requires GCC 5.3 or greater.")
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "7.0")
message(FATAL_ERROR "Requires GCC 7.0 or greater.")
set(ENV(CXX) "g++")
set(ENV(CC) "gcc")
endif()
else()
message(WARNING "You are using an unsupported compiler. Compilation has only been tested with Clang and GCC.")
message(WARNING "You are using an unsupported compiler (${CMAKE_CXX_COMPILER_ID}). Compilation has only been tested with Clang and GCC.")
endif()

include(CheckCXXCompilerFlag)
check_cxx_compiler_flag("-fvisibility-inlines-hidden" SUPPORTS_FVISIBILITY_INLINES_HIDDEN_FLAG)
if(SUPPORTS_FVISIBILITY_INLINES_HIDDEN_FLAG)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility-inlines-hidden")
endif()
check_cxx_compiler_flag("-fvisibility=hidden" SUPPORTS_FVISIBILITY_FLAG)
if(SUPPORTS_FVISIBILITY_FLAG)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden")
endif()


Expand All @@ -68,6 +107,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL DARWIN)
SET(KDU_MAKE "Makefile-MAC-x86-${BITS}-gcc")
SET(KDU_ARCH "Mac-x86-${BITS}-gcc")
SET(LUA_ARCH "macosx")
# ToDo: shouldn't this be also set on the mac through FindOpenSSL?
SET(OPENSSL_ROOT_DIR "/usr/local/opt/openssl" CACHE FILEPATH "openssl")
SET(OPENSSL_USE_STATIC_LIBS ON)
elseif(CMAKE_SYSTEM_NAME STREQUAL LINUX)
Expand Down Expand Up @@ -95,7 +135,6 @@ else()
message(STATUS "No SSL support found!")
endif()


#SET(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
if(MAKE_SHARED_SIPI)
SET(BUILD_SHARED_LIBRARIES ON)
Expand All @@ -107,8 +146,9 @@ else()
endif()
endif()



#
# set path to special find_package() code
#
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")

if(MAKE_DEBUG)
Expand All @@ -117,27 +157,24 @@ else()
set(CMAKE_CXX_FLAGS "-O3 -DNDEBUG -std=c++11 -Wall -Wno-uninitialized -Wno-deprecated -Woverloaded-virtual")
endif()

set(COMMON_SOURCE_FILES_DIR ${PROJECT_SOURCE_DIR}/src)
set(COMMON_INCLUDE_FILES_DIR ${PROJECT_SOURCE_DIR}/include)
set(DOXYGEN_FILES_DIR ${PROJECT_SOURCE_DIR}/doxygen)
set(EXT_PROJECTS_DIR ${PROJECT_SOURCE_DIR}/ext)
set(COMMON_LOCAL ${PROJECT_SOURCE_DIR}/local)
set(COMMON_SRCS ${PROJECT_SOURCE_DIR}/extsrcs)
set(COMMON_PATCHES ${PROJECT_SOURCE_DIR}/patches)
set(COMMON_VENDOR ${PROJECT_SOURCE_DIR}/vendor)
set(SIPI_SOURCE_DIR ${PROJECT_SOURCE_DIR})
set(CMAKE_INSTALL_PREFIX ${COMMON_LOCAL})
set(CONFIGURE_LIBDIR ${CMAKE_INSTALL_PREFIX}/lib)
set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_PREFIX}/lib)
include(CheckCXXCompilerFlag)
check_cxx_compiler_flag("-fvisibility-inlines-hidden" SUPPORTS_FVISIBILITY_INLINES_HIDDEN_FLAG)
if(SUPPORTS_FVISIBILITY_INLINES_HIDDEN_FLAG)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility-inlines-hidden")
endif()
check_cxx_compiler_flag("-fvisibility=hidden" SUPPORTS_FVISIBILITY_FLAG)
if(SUPPORTS_FVISIBILITY_FLAG)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden")
endif()



#
# libmagic
# (used to be build locally from source was moved to prerequisites)
#
find_package(LibMagic REQUIRED)



#find_package(JBIG REQUIRED)
#include_directories(${JBIG_INCLUDE_DIR})
# message(STATUS "JBIG ${JBIG_INCLUDE_DIR}")
Expand Down Expand Up @@ -175,28 +212,32 @@ endif()
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)

# !!! order is important !!!
# dependent targets need to be added before the dependers,
# e.g., png, bzip2 before freetype
add_subdirectory(${EXT_PROJECTS_DIR}/xz)
add_subdirectory(${EXT_PROJECTS_DIR}/zlib)
add_subdirectory(${EXT_PROJECTS_DIR}/jpeg)
add_subdirectory(${EXT_PROJECTS_DIR}/jbigkit)
add_subdirectory(${EXT_PROJECTS_DIR}/tiff)
add_subdirectory(${EXT_PROJECTS_DIR}/expat)
#add_subdirectory(${EXT_PROJECTS_DIR}/fontconfig)
#add_subdirectory(${EXT_PROJECTS_DIR}/freetype2)
add_subdirectory(${EXT_PROJECTS_DIR}/png)
add_subdirectory(${EXT_PROJECTS_DIR}/bzip2)
add_subdirectory(${EXT_PROJECTS_DIR}/freetype)
add_subdirectory(${EXT_PROJECTS_DIR}/harfbuzz)
add_subdirectory(${EXT_PROJECTS_DIR}/fontconfig)
add_subdirectory(${EXT_PROJECTS_DIR}/unistring)
add_subdirectory(${EXT_PROJECTS_DIR}/podofo)
add_subdirectory(${EXT_PROJECTS_DIR}/gettext)
add_subdirectory(${EXT_PROJECTS_DIR}/lcms2)
add_subdirectory(${EXT_PROJECTS_DIR}/exiv2)
add_subdirectory(${EXT_PROJECTS_DIR}/png)
add_subdirectory(${EXT_PROJECTS_DIR}/jansson)
add_subdirectory(${EXT_PROJECTS_DIR}/lua)
add_subdirectory(${EXT_PROJECTS_DIR}/luarocks)
add_subdirectory(${EXT_PROJECTS_DIR}/sqlite3)
add_subdirectory(${EXT_PROJECTS_DIR}/kakadu)
add_subdirectory(${EXT_PROJECTS_DIR}/curl)

#add_subdirectory(${EXT_PROJECTS_DIR}/podofo)



add_subdirectory(${EXT_PROJECTS_DIR}/poppler)

# add a target to generate API documentation with Doxygen
find_package(Doxygen)
Expand All @@ -208,9 +249,7 @@ if(DOXYGEN_FOUND)
)
endif(DOXYGEN_FOUND)




# get version from git
IF(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/.git)
FIND_PACKAGE(Git)
IF(GIT_FOUND)
Expand Down Expand Up @@ -246,8 +285,7 @@ CONFIGURE_FILE(${COMMON_INCLUDE_FILES_DIR}/SipiConfig.h.in ${COMMON_INCLUDE_FILE
# file(DOWNLOAD http://download.adobe.com/pub/adobe/iccprofiles/win/AdobeICCProfilesWin_end-user.zip ${COMMON_INCLUDE_FILES_DIR}/icc.zip SHOW_PROGRESS)
#endif()

set(
GENERATED_ICC_HEADERS
set(GENERATED_ICC_HEADERS
${COMMON_INCLUDE_FILES_DIR}/AdobeRGB1998_icc.h
${COMMON_INCLUDE_FILES_DIR}/USWebCoatedSWOP_icc.h
${COMMON_INCLUDE_FILES_DIR}/YCC709_icc.h
Expand Down Expand Up @@ -285,35 +323,34 @@ add_custom_command(
# COMMAND rm ${COMMON_INCLUDE_FILES_DIR}/USWebCoatedSWOP.icc
# COMMAND rm -rf '${COMMON_INCLUDE_FILES_DIR}/Adobe ICC Profiles \(end-user\)'
#)
add_custom_target(
icc_profiles
add_custom_target(icc_profiles
DEPENDS ${GENERATED_ICC_HEADERS}
)

link_directories(
/usr/local/lib
${CMAKE_CURRENT_SOURCE_DIR}/local/lib
${CONFIGURE_LIBDIR}
)

include_directories(
${COMMON_INCLUDE_FILES_DIR}
${COMMON_INCLUDE_FILES_DIR}/metadata
${COMMON_INCLUDE_FILES_DIR}/formats
${COMMON_INCLUDE_FILES_DIR}/iiifparser
${PROJECT_SOURCE_DIR}/shttps
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/local/include
${CMAKE_CURRENT_SOURCE_DIR}/local/include/openjpeg-2.1
/usr/local/include
${COMMON_INCLUDE_FILES_DIR}
${COMMON_INCLUDE_FILES_DIR}/metadata
${COMMON_INCLUDE_FILES_DIR}/formats
${COMMON_INCLUDE_FILES_DIR}/iiifparser
${PROJECT_SOURCE_DIR}/shttps
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/local/include
${CMAKE_CURRENT_SOURCE_DIR}/local/include/openjpeg-2.1
${COMMON_LOCAL}/include
/usr/local/include
)

add_executable(
sipi
add_executable(sipi
src/sipi.cpp
src/SipiConf.cpp include/SipiConf.h
src/SipiError.cpp include/SipiError.h
include/AdobeRGB1998_icc.h include/USWebCoatedSWOP_icc.h
include/CLI11.hpp
src/metadata/SipiIcc.cpp include/metadata/SipiIcc.h
src/metadata/SipiXmp.cpp include/metadata/SipiXmp.h
src/metadata/SipiIptc.cpp include/metadata/SipiIptc.h
Expand All @@ -324,6 +361,7 @@ add_executable(
src/formats/SipiIOJ2k.cpp include/formats/SipiIOJ2k.h
src/formats/SipiIOJpeg.cpp include/formats/SipiIOJpeg.h
src/formats/SipiIOPng.cpp include/formats/SipiIOPng.h
src/formats/SipiIOPdf.cpp include/formats/SipiIOPdf.h
src/SipiHttpServer.cpp include/SipiHttpServer.h
src/SipiCache.cpp include/SipiCache.h
src/SipiLua.cpp include/SipiLua.h
Expand All @@ -343,15 +381,49 @@ add_executable(
shttps/Parsing.cpp shttps/Parsing.h
shttps/Server.cpp shttps/Server.h
shttps/jwt.c shttps/jwt.h
shttps/makeunique.h src/SipiFilenameHash.cpp include/SipiFilenameHash.h)
add_dependencies(sipi icc_profiles)


shttps/makeunique.h src/SipiFilenameHash.cpp include/SipiFilenameHash.h src/formats/SipiIOPdf.cpp include/formats/SipiIOPdf.h include/iiifparser/SipiIdentifier.h src/iiifparser/SipiIdentifier.cpp)

add_dependencies(sipi
icc_profiles)


# !!! ORDER MATTERS !!!
target_link_libraries(sipi ${LIBS}
lcms2
exiv2
exiv2-xmp
expat
tiff
jbigkit
png
kdu_aux
kdu
xz
magic
lua
jansson
sqlite3
dl
pthread
curl
poppler-cpp
poppler
podofo
idn
jpeg
fontconfig
harfbuzz
freetype
bzip2
unistring
${CMAKE_DL_LIBS}
z
m)

# MESSAGE(STATUS "LIBS: " ${LIBS})
target_link_libraries(sipi ${LIBS} lcms2 exiv2 expat jpeg tiff jbigkit png kdu_aux kdu xz magic lua jansson sqlite3 dl pthread curl ${CMAKE_DL_LIBS} z m)
if(CMAKE_SYSTEM_NAME STREQUAL DARWIN)
target_link_libraries(sipi iconv)
target_link_libraries(sipi
iconv
gettext_intl)
else()
target_link_libraries(sipi rt)
endif()
Expand All @@ -361,8 +433,6 @@ if(OPENSSL_FOUND)
endif()




install(TARGETS sipi
RUNTIME DESTINATION bin
LIBRARY DESTINATION lib)
Expand All @@ -386,14 +456,14 @@ install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/config/sipi.config.lua
enable_testing()

# custom target to only run 'e2e' tests
add_custom_target(check
add_custom_target(e2e
DEPENDS sipi
COMMAND pytest
COMMAND pytest --sipi-exec=${CMAKE_BINARY_DIR}/sipi
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/test/e2e)

# enables running of 'e2e' tests when `make test` is executed
add_test(NAME all_python_based_e2e_tests
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/test/e2e
COMMAND pytest)
COMMAND pytest --sipi-exec=$<TARGET_FILE:sipi>)

add_subdirectory(test/unit)
Loading

0 comments on commit 6f46892

Please sign in to comment.