diff --git a/scripts/docker/debian/stretch/Dockerfile.doc b/scripts/docker/debian/stretch/Dockerfile.doc index 477b6285732..980522d7364 100644 --- a/scripts/docker/debian/stretch/Dockerfile.doc +++ b/scripts/docker/debian/stretch/Dockerfile.doc @@ -10,12 +10,15 @@ RUN apt-get update && apt-get -y install \ RUN apt-get -y install \ doxygen \ graphviz \ + ruby \ + ruby-dev \ ruby-ronn \ sloccount \ texlive-latex-base \ texlive-latex-recommended \ texlive-latex-extra \ texlive-fonts-recommended \ + && gem install apiaryio \ && rm -rf /var/lib/apt/lists/* # Create User:Group diff --git a/scripts/docker/homepage/backend/Dockerfile b/scripts/docker/homepage/backend/Dockerfile new file mode 100644 index 00000000000..32d6e004f73 --- /dev/null +++ b/scripts/docker/homepage/backend/Dockerfile @@ -0,0 +1,112 @@ +FROM debian:stretch +ENV LANG C.UTF-8 +ENV LANGUAGE C.UTF-8 +ENV LC_ALL C.UTF-8 + +RUN apt-get update && apt-get -y install \ + cmake git build-essential curl \ + libboost-all-dev \ + libpcre3-dev \ + zlib1g-dev \ + libgcrypt11-dev \ + libicu-dev \ + python \ + libssl-dev \ + libyajl-dev \ + autoconf \ + automake \ + pkg-config \ + net-tools \ + && rm -rf /var/lib/apt/lists/* + +ARG PARALLEL=2 +WORKDIR /app/deps +ARG CPPCMS_VERSION="1.2.0" +RUN curl -o cppcms-${CPPCMS_VERSION}.tar.bz -L \ + "https://sourceforge.net/projects/cppcms/files/cppcms/${CPPCMS_VERSION}/cppcms-${CPPCMS_VERSION}.tar.bz2/download" \ + && tar -xjvf cppcms-${CPPCMS_VERSION}.tar.bz \ + && mkdir cppcms-${CPPCMS_VERSION}/build \ + && cd cppcms-${CPPCMS_VERSION}/build \ + && cmake .. \ + && make -j ${PARALLEL} \ + && make install \ + && cd /app/deps \ + && rm -Rf cppcms-${CPPCMS_VERSION} +ARG JANSSON_VERSION="2.11" +RUN curl -O http://www.digip.org/jansson/releases/jansson-${JANSSON_VERSION}.tar.gz \ + && tar -xvzf jansson-${JANSSON_VERSION}.tar.gz \ + && cd jansson-${JANSSON_VERSION} \ + && mkdir build && cd build \ + && cmake .. \ + && make -j ${PARALLEL} \ + && make check \ + && make install \ + && cd /app/deps \ + && rm -Rf jansson-${JANSSON_VERSION} +ARG LIBJWT_VERSION="1.9.0" +RUN curl -o libjwt-v${LIBJWT_VERSION}.tar.gz -L \ + "https://github.com/benmcollins/libjwt/archive/v${LIBJWT_VERSION}.tar.gz" \ + && tar -xvzf libjwt-v${LIBJWT_VERSION}.tar.gz \ + && cd libjwt-${LIBJWT_VERSION} \ + && autoreconf -i \ + && ./configure \ + && make -j ${PARALLEL} all \ + && make install \ + && cd /app/deps \ + && rm -Rf libjwt-${LIBJWT_VERSION} + +# start build of kdb +ENV C_FLAGS="-D_FORTIFY_SOURCE=2 -Wformat -Werror=format-security -fstack-protector-strong -Wstack-protector -fPIE -pie" +WORKDIR /app/kdb +ADD . /app/kdb/ +RUN ldconfig \ + && mkdir build \ + && cd build \ + && cmake -DENABLE_ASAN=ON -DBUILD_FULL=OFF -DBUILD_SHARED=ON \ + -DBUILD_STATIC=OFF -DBUILD_DOCUMENTATION=OFF \ + -DINSTALL_SYSTEM_FILES=OFF \ + -DPLUGINS="ALL;-EXPERIMENTAL;-fstab;-semlock;-ruby;-lua;-python;-xerces;-yamlcpp;-python2;file;camel;yajl" \ + -DTOOLS="kdb;rest-backend" \ + -DCMAKE_C_FLAGS="$C_FLAGS" \ + -DCMAKE_CXX_FLAGS="$C_FLAGS" \ + -DCMAKE_EXE_LINKER_FLAGS="-Wl,-z,now -Wl,-z,relro" \ + .. \ + && make -j ${PARALLEL} \ + && ctest -T Test --output-on-failure -j ${PARALLEL} -LE kdbtests \ + && make install + + +FROM debian:stretch +ENV LANG C.UTF-8 +ENV LANGUAGE C.UTF-8 +ENV LC_ALL C.UTF-8 + +COPY --from=0 /usr/local /usr/local +RUN echo '/usr/local/lib/elektra/' > /etc/ld.so.conf.d/elektra.conf \ + && ldconfig +RUN apt-get update && apt-get install -y \ + libasan3 \ + libubsan0 \ + libboost-system1.62.0 \ + libboost-filesystem1.62.0 \ + libboost-thread1.62.0 \ + libssl1.1 \ + libicu57 \ + pwgen \ + && rm -rf /var/lib/apt/lists/* + +# prepare +# asan errors in mount-rest-backend-config and run-rest-backend +RUN kdb global-mount \ + && kdb mount-rest-backend-config || /bin/true \ + && kdb set -N system /sw/elektra/restbackend/#0/current/backend/jwt/encryption/secret `pwgen -1cns 30` \ + && kdb set -N system /sw/elektra/restbackend/#0/current/cppcms/service/api "http" \ + && kdb set -N system /sw/elektra/restbackend/#0/current/cppcms/service/ip "0.0.0.0" \ + && kdb set -N system /sw/elektra/restbackend/#0/current/cppcms/service/port 8080 \ + && kdb set -N system /sw/elektra/restbackend/#0/current/cppcms/http/script_names/#0 "/" \ + && kdb set '/sw/elektra/restbackend/#0/current/cppcms/daemon/enable' '0'\ + && kdb set '/sw/elektra/restbackend/#0/current/cppcms/logging/level' 'debug' + +ENTRYPOINT ["kdb"] +CMD ["run-rest-backend"] +EXPOSE 8080 diff --git a/scripts/docker/homepage/frontend/Dockerfile b/scripts/docker/homepage/frontend/Dockerfile new file mode 100644 index 00000000000..433a339f906 --- /dev/null +++ b/scripts/docker/homepage/frontend/Dockerfile @@ -0,0 +1,46 @@ +FROM debian:stretch + +ENV LANG C.UTF-8 +ENV LANGUAGE C.UTF-8 +ENV LC_ALL C.UTF-8 + +RUN apt-get update && apt-get -y install \ + cmake git build-essential curl \ + libyajl-dev \ + && curl -sL https://deb.nodesource.com/setup_9.x | bash - \ + && apt-get install -y nodejs \ + && rm -rf /var/lib/apt/lists/* + +ARG PARALLEL=2 +# start build of kdb +ENV C_FLAGS="-D_FORTIFY_SOURCE=2 -Wformat -Werror=format-security -fstack-protector-strong -Wstack-protector -fPIE -pie" +WORKDIR /app/kdb +ADD . /app/kdb/ +RUN mkdir build \ + && cd build \ + && cmake -DENABLE_ASAN=ON -DBUILD_FULL=OFF -DBUILD_SHARED=ON \ + -DBUILD_STATIC=OFF -DBUILD_DOCUMENTATION=OFF \ + -DINSTALL_SYSTEM_FILES=OFF \ + -DPLUGINS="ALL;-EXPERIMENTAL;-fstab;-semlock;-ruby;-lua;-python;-xerces;-yamlcpp;-python2;file;camel;yajl" \ + -DTOOLS="kdb;rest-frontend" \ + -DCMAKE_C_FLAGS="$C_FLAGS" \ + -DCMAKE_CXX_FLAGS="$C_FLAGS" \ + -DCMAKE_EXE_LINKER_FLAGS="-Wl,-z,now -Wl,-z,relro" \ + .. \ + && make -j ${PARALLEL} \ + && ctest -T Test --output-on-failure -j ${PARALLEL} -LE kdbtests \ + && make install \ + && echo '/usr/local/lib/elektra/' > /etc/ld.so.conf.d/elektra.conf \ + && ldconfig + +ARG BACKEND=https://restapi.libelektra.org/ +ARG FRONTEND=https://libelektra.org/ +RUN kdb global-mount \ + && kdb mount-rest-frontend-config \ + && kdb set -N system /sw/elektra/restfrontend/#0/current/backend/root "${BACKEND}" \ + && kdb set -N system /sw/elektra/restfrontend/#0/current/website/url "${URL}" \ + && kdb build-rest-frontend + +FROM nginx:alpine +COPY --from=0 /usr/local/share/elektra/tool_data/rest-frontend/public \ + /usr/share/nginx/html diff --git a/scripts/jenkins/Jenkinsfile b/scripts/jenkins/Jenkinsfile index a4e96d0186d..68cbb4492d6 100644 --- a/scripts/jenkins/Jenkinsfile +++ b/scripts/jenkins/Jenkinsfile @@ -32,37 +32,56 @@ properties([ // Globals docker_node_label = 'docker' +registry = 'hub.libelektra.org' DOCKER_IMAGES = [ sid: [ id: prefixDockerId('debian-sid'), context: "./scripts/docker/debian/sid", - file: "./scripts/docker/debian/sid/Dockerfile" + file: "./scripts/docker/debian/sid/Dockerfile", + autobuild: true ], stretch: [ id: prefixDockerId('debian-stretch'), context: "./scripts/docker/debian/stretch", file: "./scripts/docker/debian/stretch/Dockerfile", + autobuild: true ], stretch_minimal: [ id: prefixDockerId('debian-stretch-minimal'), context: "./scripts/docker/debian/stretch", file: "./scripts/docker/debian/stretch/Dockerfile.minimal", + autobuild: true ], stretch_doc: [ id: prefixDockerId('debian-stretch-doc'), context: "./scripts/docker/debian/stretch", file: "./scripts/docker/debian/stretch/Dockerfile.doc", + autobuild: true ], jessie: [ id: prefixDockerId('debian-jessie'), context: "./scripts/docker/debian/jessie", file: "./scripts/docker/debian/jessie/Dockerfile", + autobuild: true ], xenial: [ id: prefixDockerId('ubuntu-xenial'), context: "./scripts/docker/ubuntu/xenial", file: "./scripts/docker/ubuntu/xenial/Dockerfile", + autobuild: true + ], + homepage_frontend: [ + id: prefixDockerId('homepage-frontend'), + context: ".", + file: "./scripts/docker/homepage/frontend/Dockerfile", + autobuild: false + ], + homepage_backend: [ + id: prefixDockerId('homepage-backend'), + context: ".", + file: "./scripts/docker/homepage/backend/Dockerfile", + autobuild: false ] ] @@ -134,10 +153,51 @@ stage("Full builds") { parallel generate_full_build_stages() } -// build debian packages + upload if on master -stage("Build Debian Packages") { - milestone label: "Debian Packages" - parallel generate_package_build_stages() +// build artifacts (debian packages, homepage, ...) +stage("Build artifacts") { + milestone label: "artifacts" + parallel generate_artifact_stages() +} + +//maybeStage("Deploy Homepage", env.BRANCH_NAME=="master") { TODO replace +maybeStage("Deploy Homepage", true) { + node("frontend") { + docker.withRegistry("https://${registry}", + 'docker-hub-elektra-jenkins') { + def backend_name = "elektra-backend" + def frontend_name = "elektra-frontend" + def backend = docker.image( + namingStrategyHomepage(DOCKER_IMAGES.homepage_backend) + ) + def frontend = docker.image( + namingStrategyHomepage(DOCKER_IMAGES.homepage_frontend) + ) + backend.pull() + frontend.pull() + + sh "docker stop -t 5 ${backend_name} || /bin/true" + sh "docker rm ${backend_name} || /bin/true" + backend.run("""\ + -e VIRTUAL_HOST=restapi.libelektra.org \ + -e LETSENCRYPT_HOST=restapi.libelektra.org \ + -e LETSENCRYPT_EMAIL=jenkins@hub.libelektra.org \ + --name ${backend_name} \ + --network=frontend_default \ + --restart=always""" + ) + + sh "docker stop -t 5 ${frontend_name} || /bin/true" + sh "docker rm ${frontend_name} || /bin/true" + frontend.run("""\ + -e VIRTUAL_HOST=www.libelektra.org \ + -e LETSENCRYPT_HOST=www.libelektra.org \ + -e LETSENCRYPT_EMAIL=jenkins@hub.libelektra.org \ + --name ${frontend_name} \ + --network=frontend_default \ + --restart=always""" + ) + } + } } /***************************************************************************** @@ -146,24 +206,28 @@ stage("Build Debian Packages") { def generate_docker_build_stages() { def tasks = [:] - DOCKER_IMAGES.each { id, image -> - tasks << maybe_build_image(image) + DOCKER_IMAGES.each { key, image -> + if(image.autobuild) { + tasks << maybe_build_image_stage(image) + } } return tasks } -/* Build image if not available in docker repository +/* Returns a stage that builds an image if not available in docker repository * @param image Map identifying which image to build + * @param namingStrategy describes how the image should be named + see namingStrategyTesting/namingStrategyHomepage */ -def maybe_build_image(image) { +def maybe_build_image_stage(image, namingStrategy=this.&namingStrategyTesting) { def taskname = "build/${image.id}/" return [(taskname): { stage(taskname) { node(docker_node_label) { echo "Starting ${env.STAGE_NAME} on ${env.NODE_NAME}" checkout scm - def id = imageFullName(image) - docker.withRegistry('https://hub.libelektra.org', + def id = namingStrategy(image) + docker.withRegistry("https://${registry}", 'docker-hub-elektra-jenkins') { try { // check if Docker file with tag exists @@ -172,11 +236,14 @@ def maybe_build_image(image) { // build and push if it does not def uid = getUid() def gid = getGid() + def cpu_count = cpuCount() + def i = docker.build( id,"""\ --pull \ --build-arg JENKINS_GROUPID=${gid} \ --build-arg JENKINS_USERID=${uid} \ +--build-arg PARALLEL=${cpu_count} \ -f ${image.file} ${image.context}""" ) i.push() @@ -298,9 +365,20 @@ def build_doc() { cmake(env.WORKSPACE, cmake_flags) sh "make html latex man pdf" } + + def apib = "./doc/api_blueprints/snippet-sharing.apib" + def api_doc_dir = "./build/API_DOC/restapi" + sh "mkdir -p ${api_doc_dir}/${VERSION}" + sh "cp ${apib} ${api_doc_dir}/${VERSION}/" + apiary(apib, "${api_doc_dir}/${VERSION}/snippet-sharing.html") + dir(api_doc_dir) { + sh "ln -s ${VERSION} current" + } + warnings parserConfigurations: [ [parserName: 'Doxygen', pattern: 'build/doc/doxygen.log'] ] + // TODO don't write to latest on PR's sshPublisher(publishers: [ sshPublisherDesc( @@ -320,6 +398,10 @@ def build_doc() { sourceFiles: 'doc/help/*.html', removePrefix: 'doc/help/', remoteDirectory: 'help' + ), + sshTransfer( + sourceFiles: 'build/API_DOC/*', + removePrefix: 'build/API_DOC/' ) ]) ]) @@ -420,11 +502,12 @@ kill `pidof dbus-daemon` } } -/* Generate Stages that build Debian packages +/* Generate Stages that build and deploy artifacts */ -def generate_package_build_stages() { +def generate_artifact_stages() { def tasks = [:] tasks << build_package_debian_stretch() + tasks << build_homepage() return tasks } @@ -467,6 +550,19 @@ def build_package_debian_stretch() { } } +def build_homepage() { + def homepage_tasks = [:] + homepage_tasks << maybe_build_image_stage( + DOCKER_IMAGES.homepage_frontend, + this.&namingStrategyHomepage + ) + homepage_tasks << maybe_build_image_stage( + DOCKER_IMAGES.homepage_backend, + this.&namingStrategyHomepage + ) + return homepage_tasks +} + /***************************************************************************** * Define helper functions *****************************************************************************/ @@ -572,7 +668,7 @@ def withDockerEnv(stage_name, image, cl) { return [(stage_name): { stage(stage_name) { node(docker_node_label) { - docker.withRegistry('https://hub.libelektra.org', + docker.withRegistry("https://${registry}", 'docker-hub-elektra-jenkins') { timeout(activity: true, time: 5, unit: 'MINUTES') { def cpu_count = cpuCount() @@ -580,7 +676,7 @@ def withDockerEnv(stage_name, image, cl) { "CTEST_PARALLEL_LEVEL='${cpu_count+2}'"]) { echo "Starting ${env.STAGE_NAME} on ${env.NODE_NAME}" checkout scm - docker.image(imageFullName(image)) + docker.image(namingStrategyTesting(image)) .inside("-v ${env.HOME}/git_mirrors:/home/jenkins/git_mirrors") { cl() } } } @@ -637,18 +733,26 @@ def xunitUpload() { ]) } -/* Build full name of docker image +/* Build image ID of docker images used for tests * * We use identifiers in the form of name:yyyyMM-hash * The hash is build from reading the Dockerfile * @param image_map Map identifying an docker image (see DOCKER_IMAGES) */ -def imageFullName(image_map) { +def namingStrategyTesting(image_map) { def cs = checksum(image_map.file) def dateString = dateFormatter(NOW) return "${image_map.id}:${dateString}-${cs}" } +/* Build image ID for homepage + * + * @param image_map Map identifying an docker image + */ +def namingStrategyHomepage(image_map) { + return "${image_map.id}:${BUILD_NUMBER}" +} + /* Generate the checksum of a file * @param file File to generate a checksum for */ @@ -690,7 +794,7 @@ def dateFormatter(date) { * @param name Name of the docker image */ def prefixDockerId(name) { - def base = 'build-elektra-' + def base = "${registry}/build-elektra-" return base + name } @@ -749,3 +853,11 @@ def publishDebianPackages(remote="a7") { echo "Skipping package publish because we are not on master" } } + +/* Run apiary + * @param input Input file (.apib) + * @param output Output file (.html) + */ +def apiary(input, output) { + sh "apiary preview --path=${input} --output=${output}" +} diff --git a/src/tools/rest-backend/run.sh b/src/tools/rest-backend/run.sh index 470e44082ab..0de7323cd91 100755 --- a/src/tools/rest-backend/run.sh +++ b/src/tools/rest-backend/run.sh @@ -1,2 +1,2 @@ #!/bin/sh -@CMAKE_INSTALL_PREFIX@/@install_directory@/@tool@ +exec @CMAKE_INSTALL_PREFIX@/@install_directory@/@tool@