diff --git a/technologies/app/rstudio/metadata.yaml b/technologies/app/rstudio/metadata.yaml index 3230560c3..899952239 100644 --- a/technologies/app/rstudio/metadata.yaml +++ b/technologies/app/rstudio/metadata.yaml @@ -67,7 +67,7 @@ contexts: releaseNotes: First version of RStudio 4.2 into Saagie. available: true trustLevel: stable - recommended: true + recommended: false ports: - port: 80 name: Rstudio @@ -77,4 +77,20 @@ contexts: dockerInfo: image: "saagie/rstudio" baseTag: "4.2" - version: "4.2-1.141.0_SDKTECHNO-211" \ No newline at end of file + version: "4.2-1.141.0_SDKTECHNO-211" + - id: rstudio-4.3 + label: "4.3" + releaseNotes: First version of RStudio 4.3 into Saagie. + available: true + trustLevel: stable + recommended: true + ports: + - port: 80 + name: Rstudio + rewriteUrl: false + basePath: SAAGIE_BASE_PATH + volumes: ["/home"] + dockerInfo: + image: "saagie/rstudio" + baseTag: "4.3" + version: "4.3-1.163.0_sdktechno-238" \ No newline at end of file diff --git a/technologies/app/rstudio/rstudio-4.2/context.yaml b/technologies/app/rstudio/rstudio-4.2/context.yaml index 8659d678c..3d26b2793 100644 --- a/technologies/app/rstudio/rstudio-4.2/context.yaml +++ b/technologies/app/rstudio/rstudio-4.2/context.yaml @@ -3,7 +3,7 @@ label: "4.2" releaseNotes: First version of RStudio 4.2 into Saagie. available: true trustLevel: stable -recommended: true +recommended: false ports: - port: 80 name: Rstudio diff --git a/technologies/app/rstudio/rstudio-4.3/Dockerfile b/technologies/app/rstudio/rstudio-4.3/Dockerfile new file mode 100644 index 000000000..1eaec4d9f --- /dev/null +++ b/technologies/app/rstudio/rstudio-4.3/Dockerfile @@ -0,0 +1,25 @@ +ARG base_img + +FROM ${base_img} AS BASE_IMG + +# Nginx isntall and remove default conf +RUN apt-get update -qq && apt-get install -yqq --no-install-recommends nginx \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/ \ + && rm /etc/nginx/sites-enabled/default +COPY resources/server.conf /etc/nginx/sites-enabled/rstudio.conf +COPY resources/userconf /etc/cont-init.d/userconf +COPY resources/init_rstudio.sh /init_rstudio.sh + +# Be sure rstudio user has full access to his home directory +RUN mkdir -p /home/rstudio && \ + chown -R rstudio:rstudio /home/rstudio && \ + chmod -R 755 /home/rstudio && \ + chmod 500 /init_rstudio.sh + +EXPOSE 80 + +COPY resources/entrypoint.sh /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] + +CMD ["/bin/sh", "-c", "/init_rstudio.sh"] diff --git a/technologies/app/rstudio/rstudio-4.3/build.gradle.kts b/technologies/app/rstudio/rstudio-4.3/build.gradle.kts new file mode 100644 index 000000000..036a6e380 --- /dev/null +++ b/technologies/app/rstudio/rstudio-4.3/build.gradle.kts @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Copyright 2019-2021. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import com.bmuschko.gradle.docker.DockerRemoteApiPlugin +import com.saagie.technologies.SaagieTechnologiesGradlePlugin + + +apply() +apply() + + +tasks.withType(com.bmuschko.gradle.docker.tasks.image.DockerBuildImage::class) { + this.buildArgs.put( + "base_img", + "saagie/r-image:4.3-1.163.0_sdktechno-238" + ) +} diff --git a/technologies/app/rstudio/rstudio-4.3/context.yaml b/technologies/app/rstudio/rstudio-4.3/context.yaml new file mode 100644 index 000000000..9c4d38d5b --- /dev/null +++ b/technologies/app/rstudio/rstudio-4.3/context.yaml @@ -0,0 +1,12 @@ +id: rstudio-4.3 +label: "4.3" +releaseNotes: First version of RStudio 4.3 into Saagie. +available: true +trustLevel: stable +recommended: true +ports: + - port: 80 + name: Rstudio + rewriteUrl: false + basePath: SAAGIE_BASE_PATH +volumes: ["/home"] diff --git a/technologies/app/rstudio/rstudio-4.3/dockerInfo.yaml b/technologies/app/rstudio/rstudio-4.3/dockerInfo.yaml new file mode 100644 index 000000000..de8da9116 --- /dev/null +++ b/technologies/app/rstudio/rstudio-4.3/dockerInfo.yaml @@ -0,0 +1,4 @@ +image: saagie/rstudio +baseTag: 4.3 +dynamicVersion: 1.163.0_sdktechno-238 +version: 4.3-1.163.0_sdktechno-238 diff --git a/technologies/app/rstudio/rstudio-4.3/image_test.yaml b/technologies/app/rstudio/rstudio-4.3/image_test.yaml new file mode 100644 index 000000000..b20d3a15c --- /dev/null +++ b/technologies/app/rstudio/rstudio-4.3/image_test.yaml @@ -0,0 +1,96 @@ +schemaVersion: "2.0.0" + +metadataTest: + env: + - key: LANG + value: en_US.UTF-8 + - key: JAVA_HOME + value: /usr/lib/jvm/java-11-openjdk-amd64 + - key: HADOOP_HOME + value: /hadoop/hadoop-2.7.0 + - key: HIVE_HOME + value: /apache-hive-1.2.2-bin + +fileExistenceTests: + - name: "entrypoint" + path: "/entrypoint.sh" + shouldExist: true + permissions: "-rwxr-xr-x" + - name: "init_rstudio" + path: "/init_rstudio.sh" + shouldExist: true + permissions: "-r-x------" + - name: "rstudio_home" + path: "/home/rstudio" + shouldExist: true + permissions: "drwxr-xr-x" + uid: 1000 + gid: 1000 + - name: No default nginx conf + path: /etc/nginx/sites-enabled/default + shouldExist: false + - name: rstudio nginx conf + path: "/etc/nginx/sites-enabled/rstudio.conf" + shouldExist: true + permissions: "-rw-r--r--" + - name: Impala ODBC dependecy + path: "/opt/cloudera/impalaodbc/Setup/odbcinst.ini" + shouldExist: true + +fileContentTests: + - name: "OBBC Impala content" + path: "/opt/cloudera/impalaodbc/Setup/odbcinst.ini" + expectedContents: + [ + "Cloudera ODBC Driver for Impala 32-bit=Installed", + "Cloudera ODBC Driver for Impala 64-bit=Installed", + "Driver=/opt/cloudera/impalaodbc/lib/32/libclouderaimpalaodbc32.so", + "Driver=/opt/cloudera/impalaodbc/lib/64/libclouderaimpalaodbc64.so" + ] + - name: "RStudio conf" + path: "/etc/nginx/sites-enabled/rstudio.conf" + expectedContents: + [ + "listen 80 default_server;", + "listen \\[::\\]:80 default_server", + "location SAAGIE_BASE_PATH {", + "proxy_pass http:\\/\\/localhost:8787;", + ] + +commandTests: + - name: "Workdir" + command: "pwd" + expectedOutput: ["/"] + - name: "R version" + command: "R" + args: ["--version"] + expectedOutput: ['R version 4.3.*'] + - name: "java version" + command: "java" + args: ["-version"] + expectedError: ['openjdk version "11.*'] + - name: "java installation" + command: "which" + args: ["java"] + expectedOutput: ["/usr/lib/jvm/java-11-openjdk-amd64/bin/java"] + - name: "Hadoop installation" + command: "which" + args: ["hadoop"] + expectedOutput: ["/hadoop/hadoop-2.7.0/bin/hadoop"] + - name: "Beeline installation" + command: "which" + args: ["beeline"] + expectedOutput: ["/apache-hive-1.2.2-bin/bin/beeline"] + - name: check All libs + command: "bash" + args: [ + "-c", + " + Rscript -e 'library(tidyverse)' && + Rscript -e 'library(aws.s3)' && + Rscript -e 'library(arrow)' && + Rscript -e 'library(openxlsx)' && + Rscript -e 'library(data.table)' && + Rscript -e 'library(odbc)';", + ] + exitCode: 0 diff --git a/technologies/app/rstudio/rstudio-4.3/resources/entrypoint.sh b/technologies/app/rstudio/rstudio-4.3/resources/entrypoint.sh new file mode 100755 index 000000000..2ff277711 --- /dev/null +++ b/technologies/app/rstudio/rstudio-4.3/resources/entrypoint.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +if [[ -z ${RSTUDIO_ADMIN_PASSWORD} || -z ${RSTUDIO_PASSWORD} ]]; then + echo "ERROR : Missing environment variables. In order to work, this app needs the following environment variables set : " + echo "RSTUDIO_ADMIN_PASSWORD : Password for the user admin, with root permissions" + echo "RSTUDIO_PASSWORD : Password for the user rstudio" + exit 1 +fi + +R_HOME=$(Rscript -e 'Sys.getenv("R_HOME")' | sed -rn 's/^\[[[:digit:]]+\] "(.*)"/\1/p') + +# Change default CRAN to ENV VAR if present +if [[ -z "${R_CUSTOM_CRAN}" ]]; then + echo "No custom CRAN defined, using default, you can configure one using R_CUSTOM_CRAN env var" +else + echo "Custom CRAN used: ${R_CUSTOM_CRAN}" + sed -ri "s#(CRAN = ')[^']*#\1${R_CUSTOM_CRAN}#g" "${R_HOME}/etc/Rprofile.site" +fi + +sed -i 's:SAAGIE_BASE_PATH:'"$SAAGIE_BASE_PATH"':g' /etc/nginx/sites-enabled/rstudio.conf +nginx && /init_rstudio.sh \ No newline at end of file diff --git a/technologies/app/rstudio/rstudio-4.3/resources/init_rstudio.sh b/technologies/app/rstudio/rstudio-4.3/resources/init_rstudio.sh new file mode 100644 index 000000000..766929532 --- /dev/null +++ b/technologies/app/rstudio/rstudio-4.3/resources/init_rstudio.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# Grab runtime parameters +TEMP=`getopt -o p: --long port: -- "$@"` +eval set -- "$TEMP" + +while true ; do + case "$1" in + -p|--port) + case "$2" in + "") shift 2 ;; + *) RSTUDIO_PORT=$2 ; shift 2 ;; + esac ;; + --) shift ; break ;; + *) echo "Internal error!" ; exit 1 ;; + esac +done + +# check if rstudio needs to be run on a custom port +if [ -z $RSTUDIO_PORT ]; +then + echo "INFO: no port given. RStudio will run on default port (8787)." + export PORT0=8787 + export RSTUDIO_PORT=8787 +else + # If not already set, set a fake PORT0 variable (used in spark-env.sh) + if [ -z $PORT0 ] + then + export PORT0=$(( $RSTUDIO_PORT+1 )) + echo "WARNING: no PORT0 environment variable provided. $PORT0 will be used..." + fi + echo "www-port=$RSTUDIO_PORT" >> /etc/rstudio/rserver.conf +fi + +echo "Running RStudio on port $RSTUDIO_PORT" + +/init & + +# wait a bit for initialization to end before modifying the users... +sleep 5 + +echo "Init users" + +# We need to make rstudio user the owner of his own home directory as it will be owned by root if we mount a docker volume to /home (weird but true...) +mkdir -p /home/rstudio +chown -R rstudio:rstudio /home/rstudio +chmod -R 755 /home/rstudio + +# create an admin user who will be able to create new users directly from RStudio IDE +useradd admin --home /home/admin --create-home -p $(openssl passwd -1 ${RSTUDIO_ADMIN_PASSWORD}) --groups sudo,shadow,rstudio,staff + +# Add backupusers script for admin user +mkdir -p /home/admin +echo "sudo tar cf /home/admin/users_backup.tar /etc/passwd /etc/shadow 2> /dev/null" > /home/admin/backupusers +chown admin:admin /home/admin/backupusers +chmod 700 /home/admin/backupusers + +# then restore previous users if necessary +if [ -e /home/admin/users_backup.tar ] +then + echo "Restoring users..." + rm -rf /tmp/userrestore + mkdir -p /tmp/userrestore + + tar xf /home/admin/users_backup.tar -C /tmp/userrestore --strip-components=1 + + cat /tmp/userrestore/shadow | while read LINE + do + # echo $LINE + + USER=$(echo $LINE | awk -F':' '{print $1}') + ENCRYPTED_PASSWD=$(echo $LINE | awk -F':' '{print $2}') + + if ! grep -q "^$USER:" /etc/passwd + then + echo "- restoring $USER" + HOME_DIR=$(grep "^$USER:" /tmp/userrestore/passwd | awk -F':' '{print $6}') + echo " home dir: $HOME_DIR" + useradd $USER --home $HOME_DIR --create-home -p $ENCRYPTED_PASSWD + chown -R $USER:$USER $HOME_DIR + fi + + done +fi + +# propagate environment variables to every user +R_HOME=$(Rscript -e 'Sys.getenv("R_HOME")' | sed -rn 's/^\[[[:digit:]]+\] "(.*)"/\1/p') + +env | while read VAR +do + if ! grep -q "$VAR" /ROOT_ENV_VAR && ! echo "$VAR" | grep -q "RSTUDIO_ADMIN_PASSWORD" + then + echo $VAR >> $R_HOME/etc/Renviron.site + fi +done + +# keep the container running... +tail -f /dev/null diff --git a/technologies/app/rstudio/rstudio-4.3/resources/server.conf b/technologies/app/rstudio/rstudio-4.3/resources/server.conf new file mode 100644 index 000000000..49c2fc342 --- /dev/null +++ b/technologies/app/rstudio/rstudio-4.3/resources/server.conf @@ -0,0 +1,24 @@ +map $http_connection $upgrade_requested { + default upgrade; + '' close; +} +server { + listen 80 default_server; + listen [::]:80 default_server; + server_name _; + + client_max_body_size 2G; + + location SAAGIE_BASE_PATH { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $upgrade_requested; + proxy_read_timeout 20d; + + rewrite ^SAAGIE_BASE_PATH/(.*)$ /$1 break; + rewrite ^SAAGIE_BASE_PATH$ / break; + proxy_pass http://localhost:8787; + proxy_redirect http://localhost:8787/ $scheme://$hostSAAGIE_BASE_PATH/; + proxy_redirect https://localhost:8787/ $scheme://$hostSAAGIE_BASE_PATH/; + } +} \ No newline at end of file diff --git a/technologies/app/rstudio/rstudio-4.3/resources/userconf b/technologies/app/rstudio/rstudio-4.3/resources/userconf new file mode 100644 index 000000000..cc9206275 --- /dev/null +++ b/technologies/app/rstudio/rstudio-4.3/resources/userconf @@ -0,0 +1,114 @@ +#!/usr/bin/with-contenv bash + +## Set defaults for environmental variables in case they are undefined +USER=${USER:=rstudio} +PASSWORD=${PASSWORD:=rstudio} +USERID=${USERID:=1000} +GROUPID=${GROUPID:=1000} +ROOT=${ROOT:=FALSE} +UMASK=${UMASK:=022} + +## Make sure RStudio inherits the full path +echo "PATH=${PATH}" >> /usr/local/lib/R/etc/Renviron + +bold=$(tput bold) +normal=$(tput sgr0) + + +if [[ ${DISABLE_AUTH,,} == "true" ]] +then + mv /etc/rstudio/disable_auth_rserver.conf /etc/rstudio/rserver.conf + echo "USER=$USER" >> /etc/environment +fi + +if [ "$RSTUDIO_ADMIN_PASSWORD" == "rstudioadmin" ] +then + printf "\n\n" + tput bold + printf "\e[31mERROR\e[39m: You must set a unique ADMIN PASSWORD (not 'rstudioadmin') first! ie. define an environment variable RSTUDIO_ADMIN_PASSWORD or if manually run, run it with:\n" + printf "docker run -e RSTUDIO_ADMIN_PASSWORD=\e[92m\e[39m -p 80:80 saagie/rstudio\n" + tput sgr0 + printf "\n\n" + exit 1 +fi + +if grep --quiet "auth-none=1" /etc/rstudio/rserver.conf +then + echo "Skipping authentication as requested" +elif [ "$RSTUDIO_PASSWORD" == "rstudio" ] +then + printf "\n\n" + tput bold + printf "\e[31mERROR\e[39m: You must set a unique PASSWORD (not 'rstudio') first! e.g. run with:\n" + printf "docker run -e RSTUDIO_PASSWORD=\e[92m\e[39m -p 80:80 saagie/rstudio\n" + tput sgr0 + printf "\n\n" + exit 1 +else + export PASSWORD=`echo $RSTUDIO_PASSWORD` +fi + +if [ "$USERID" -lt 1000 ] +# Probably a macOS user, https://github.com/rocker-org/rocker/issues/205 + then + echo "$USERID is less than 1000" + check_user_id=$(grep -F "auth-minimum-user-id" /etc/rstudio/rserver.conf) + if [[ ! -z $check_user_id ]] + then + echo "minumum authorised user already exists in /etc/rstudio/rserver.conf: $check_user_id" + else + echo "setting minumum authorised user to 499" + echo auth-minimum-user-id=499 >> /etc/rstudio/rserver.conf + fi +fi + +if [ "$USERID" -ne 1000 ] +## Configure user with a different USERID if requested. + then + echo "deleting user rstudio" + userdel rstudio + echo "creating new $USER with UID $USERID" + useradd -m $USER -u $USERID + mkdir /home/$USER + chown -R $USER /home/$USER + usermod -a -G staff $USER +elif [ "$USER" != "rstudio" ] + then + ## cannot move home folder when it's a shared volume, have to copy and change permissions instead + cp -r /home/rstudio /home/$USER + ## RENAME the user + usermod -l $USER -d /home/$USER rstudio + groupmod -n $USER rstudio + usermod -a -G staff $USER + chown -R $USER:$USER /home/$USER + echo "USER is now $USER" +fi + +if [ "$GROUPID" -ne 1000 ] +## Configure the primary GID (whether rstudio or $USER) with a different GROUPID if requested. + then + echo "Modifying primary group $(id $USER -g -n)" + groupmod -g $GROUPID $(id $USER -g -n) + echo "Primary group ID is now custom_group $GROUPID" +fi + +## Add a password to user +echo "$USER:$PASSWORD" | chpasswd + +# Use Env flag to know if user should be added to sudoers +if [[ ${ROOT,,} == "true" ]] + then + adduser $USER sudo && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + echo "$USER added to sudoers" +fi + +## Change Umask value if desired +if [ "$UMASK" -ne 022 ] + then + echo "server-set-umask=false" >> /etc/rstudio/rserver.conf + echo "Sys.umask(mode=$UMASK)" >> /home/$USER/.Rprofile +fi + +## add these to the global environment so they are avialable to the RStudio user +echo "HTTR_LOCALHOST=$HTTR_LOCALHOST" >> /etc/R/Renviron.site +echo "HTTR_PORT=$HTTR_PORT" >> /etc/R/Renviron.site \ No newline at end of file diff --git a/technologies/job/r/metadata.yaml b/technologies/job/r/metadata.yaml index 22f7bb93f..24ce449d6 100644 --- a/technologies/job/r/metadata.yaml +++ b/technologies/job/r/metadata.yaml @@ -61,7 +61,7 @@ contexts: - id: "4.2" label: "4.2" available: true - recommended: true + recommended: false trustLevel: stable job: features: @@ -80,4 +80,27 @@ contexts: dockerInfo: image: "saagie/r-image" baseTag: "4.2" - version: "4.2-1.155.0" \ No newline at end of file + version: "4.2-1.155.0" + - id: "4.3" + label: "4.3" + available: true + recommended: true + trustLevel: stable + job: + features: + - type: COMMAND_LINE + label: Command line + mandatory: true + comment: Linux shell command to launch the job. + defaultValue: Rscript {file} + - type: ARTIFACT + label: Package + mandatory: false + comment: "Compatible upload file: .r, .tar(.gz), .zip" + - type: SCHEDULER + label: Scheduled + mandatory: true + dockerInfo: + image: "saagie/r-image" + baseTag: "4.3" + version: "4.3-1.163.0_sdktechno-238" \ No newline at end of file diff --git a/technologies/job/r/r-4.2/context.yaml b/technologies/job/r/r-4.2/context.yaml index 4c87ceb7d..58afbdecd 100644 --- a/technologies/job/r/r-4.2/context.yaml +++ b/technologies/job/r/r-4.2/context.yaml @@ -1,7 +1,7 @@ id: "4.2" label: "4.2" available: true -recommended: true +recommended: false trustLevel: stable job: features: diff --git a/technologies/job/r/r-4.3/Dockerfile b/technologies/job/r/r-4.3/Dockerfile new file mode 100644 index 000000000..188267888 --- /dev/null +++ b/technologies/job/r/r-4.3/Dockerfile @@ -0,0 +1,89 @@ +FROM rocker/tidyverse:4.3.1 + +LABEL maintainer="Saagie " + +USER root + +ENV DEBIAN_FRONTEND noninteractive + +ENV CDH_MAIN_VERSION 5 +ENV CDH_VERSION ${CDH_MAIN_VERSION}.13.0 +ENV HADOOP_VERSION 2.7.0 +ENV HIVE_VERSION 1.2.2 +ENV HIVE_ODBC_VERSION 2.6.4.1004 +ENV HIVE_ODBC_VERSION_FULL ${HIVE_ODBC_VERSION}-2_amd64 +ENV IMPALA_ODBC_VERSION 2.5.41.1029 +ENV IMPALA_ODBC_VERSION_FULL ${IMPALA_ODBC_VERSION}-2_amd64 + +# Store Root envvar to be able to exclude it at runtime when propagating envvars to every user +RUN env >> /ROOT_ENV_VAR && chmod 400 /ROOT_ENV_VAR + +# Install system libraries required +RUN apt-get update -qq \ + && apt-get install -yqq --no-install-recommends \ + curl \ + krb5-user \ + openjdk-11-jdk \ + unixodbc-dev \ + odbcinst \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Hive client installation +RUN wget --no-verbose http://apache.mirrors.ovh.net/ftp.apache.org/dist/hive/hive-${HIVE_VERSION}/apache-hive-${HIVE_VERSION}-bin.tar.gz \ + && tar -xzf apache-hive-${HIVE_VERSION}-bin.tar.gz \ + && rm apache-hive-${HIVE_VERSION}-bin.tar.gz +ENV HIVE_HOME=/apache-hive-${HIVE_VERSION}-bin +ENV PATH=$HIVE_HOME/bin:$PATH + +# Install Hive ODBC driver +RUN apt-get update -qq && apt-get install -yqq --no-install-recommends \ + libsasl2-modules-gssapi-mit \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && cd /tmp \ + && wget --no-verbose https://downloads.cloudera.com/connectors/ClouderaHive_ODBC_${HIVE_ODBC_VERSION}/Debian/clouderahiveodbc_${HIVE_ODBC_VERSION_FULL}.deb \ + && dpkg -i clouderahiveodbc_${HIVE_ODBC_VERSION_FULL}.deb \ + && odbcinst -i -d -f /opt/cloudera/hiveodbc/Setup/odbcinst.ini \ + && rm /tmp/clouderahiveodbc_${HIVE_ODBC_VERSION_FULL}.deb + +# Install Impala ODBC dependency +RUN cd /tmp \ + && wget --no-verbose https://downloads.cloudera.com/connectors/impala_odbc_${IMPALA_ODBC_VERSION}/Debian/clouderaimpalaodbc_${IMPALA_ODBC_VERSION_FULL}.deb \ + && dpkg -i clouderaimpalaodbc_${IMPALA_ODBC_VERSION_FULL}.deb \ + && odbcinst -i -d -f /opt/cloudera/impalaodbc/Setup/odbcinst.ini \ + && rm -rf /tmp/clouderaimpalaodbc_${IMPALA_ODBC_VERSION_FULL}.deb \ + && sed -i 's/DriverManagerEncoding=UTF-32/DriverManagerEncoding=UTF-16/g' /opt/cloudera/impalaodbc/lib/64/cloudera.impalaodbc.ini + +# JAVA_HOME define +ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 +ENV PATH=$JAVA_HOME/bin:$PATH + +# Add hdfs command line +RUN cd / \ + && mkdir hadoop \ + && cd hadoop \ + && wget --no-verbose https://archive.apache.org/dist/hadoop/core/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz \ + && tar xf hadoop-${HADOOP_VERSION}.tar.gz \ + && rm hadoop-${HADOOP_VERSION}.tar.gz \ + && rm -rf hadoop-${HADOOP_VERSION}/etc/hadoop/ \ + && ln -s /etc/hadoop/conf hadoop-${HADOOP_VERSION}/etc/hadoop +ENV HADOOP_HOME=/hadoop/hadoop-${HADOOP_VERSION} +ENV PATH=$PATH:$HADOOP_HOME/bin + +# Packages install +RUN install2.r --error --skipinstalled \ + arrow \ + openxlsx \ + odbc + +# Install aws.s3 from saagie github (fix connection to OVH) +RUN R -e "devtools::install_github('saagie/aws.s3@v0.3.22.1')" + +# Move scripts and frequently changing directive to the end of the build +COPY entrypoint.sh /entrypoint.sh +RUN chmod 755 /entrypoint.sh + +WORKDIR /sandbox + +CMD ["/bin/bash", "-c", "/entrypoint.sh"] diff --git a/technologies/job/r/r-4.3/build.gradle.kts b/technologies/job/r/r-4.3/build.gradle.kts new file mode 100644 index 000000000..b814e94a3 --- /dev/null +++ b/technologies/job/r/r-4.3/build.gradle.kts @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Copyright 2019-2021. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import com.bmuschko.gradle.docker.DockerRemoteApiPlugin +import com.saagie.technologies.SaagieTechnologiesGradlePlugin + +apply() +apply() diff --git a/technologies/job/r/r-4.3/buildMe b/technologies/job/r/r-4.3/buildMe new file mode 100644 index 000000000..0f3016682 --- /dev/null +++ b/technologies/job/r/r-4.3/buildMe @@ -0,0 +1 @@ +001 \ No newline at end of file diff --git a/technologies/job/r/r-4.3/context.yaml b/technologies/job/r/r-4.3/context.yaml new file mode 100644 index 000000000..8d5168c1b --- /dev/null +++ b/technologies/job/r/r-4.3/context.yaml @@ -0,0 +1,19 @@ +id: "4.3" +label: "4.3" +available: true +recommended: true +trustLevel: stable +job: + features: + - type: COMMAND_LINE + label: Command line + mandatory: true + comment: Linux shell command to launch the job. + defaultValue: Rscript {file} + - type: ARTIFACT + label: Package + mandatory: false + comment: "Compatible upload file: .r, .tar(.gz), .zip" + - type: SCHEDULER + label: Scheduled + mandatory: true diff --git a/technologies/job/r/r-4.3/dockerInfo.yaml b/technologies/job/r/r-4.3/dockerInfo.yaml new file mode 100644 index 000000000..20e8271cc --- /dev/null +++ b/technologies/job/r/r-4.3/dockerInfo.yaml @@ -0,0 +1,4 @@ +image: saagie/r-image +baseTag: 4.3 +dynamicVersion: 1.163.0_sdktechno-238 +version: 4.3-1.163.0_sdktechno-238 diff --git a/technologies/job/r/r-4.3/entrypoint.sh b/technologies/job/r/r-4.3/entrypoint.sh new file mode 100644 index 000000000..f0cb23ddd --- /dev/null +++ b/technologies/job/r/r-4.3/entrypoint.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -euo pipefail + +R_HOME=$(Rscript -e 'Sys.getenv("R_HOME")' | sed -rn 's/^\[[[:digit:]]+\] "(.*)"/\1/p') + +# Change default CRAN to ENV VAR if present +if [[ "${R_CUSTOM_CRAN:-empty}" == "empty" ]]; then + echo "No custom CRAN defined, using default, you can configure one using R_CUSTOM_CRAN env var." +else + echo "Custom CRAN used: ${R_CUSTOM_CRAN}" + sed -ri "s#(CRAN = ')[^']*#\1${R_CUSTOM_CRAN}#g" "${R_HOME}/etc/Rprofile.site" +fi + +if compgen -G "*.zip" > /dev/null; then + echo "Zip file detected ... unzipping ..." + unzip -qo *.zip +elif compgen -G "*.tar.gz" > /dev/null; then + echo "Tar gz file detected ... unzipping ..." + tar -xf *.tar.gz +elif compgen -G "*.tar" > /dev/null; then + echo "Tar file detected ... unzipping ..." + tar -xf *.tar +fi + +if test -f main_script; then + echo "'main_script' detected ... executing ..." + bash ./main_script +else + exec "$@" +fi diff --git a/technologies/job/r/r-4.3/image_test.yaml b/technologies/job/r/r-4.3/image_test.yaml new file mode 100644 index 000000000..cc5a777bb --- /dev/null +++ b/technologies/job/r/r-4.3/image_test.yaml @@ -0,0 +1,78 @@ +schemaVersion: "2.0.0" + +metadataTest: + env: + - key: LANG + value: en_US.UTF-8 + - key: JAVA_HOME + value: /usr/lib/jvm/java-11-openjdk-amd64 + - key: HADOOP_HOME + value: /hadoop/hadoop-2.7.0 + - key: HIVE_HOME + value: /apache-hive-1.2.2-bin + +fileExistenceTests: + - name: "kinit" + path: "/usr/bin/kinit" + shouldExist: true + permissions: "-rwxr-xr-x" + - name: "entrypoint.sh" + path: "/entrypoint.sh" + shouldExist: true + permissions: "-rwxr-xr-x" + - name: Impala ODBC dependecy + path: "/opt/cloudera/impalaodbc/Setup/odbcinst.ini" + shouldExist: true + +fileContentTests: + - name: "OBBC Impala content" + path: "/opt/cloudera/impalaodbc/Setup/odbcinst.ini" + expectedContents: + [ + "Cloudera ODBC Driver for Impala 32-bit=Installed", + "Cloudera ODBC Driver for Impala 64-bit=Installed", + "Driver=/opt/cloudera/impalaodbc/lib/32/libclouderaimpalaodbc32.so", + "Driver=/opt/cloudera/impalaodbc/lib/64/libclouderaimpalaodbc64.so" + ] + +commandTests: + - name: "R version" + command: "R" + args: ["--version"] + expectedOutput: ['R version 4.3.*'] + - name: "krb5-user installation" + command: "kinit" + expectedError: ['kinit: Program lacks support for encryption type while getting initial credentials'] + exitCode: 1 + - name: "Workdir" + command: "pwd" + expectedOutput: ["/sandbox"] + - name: "java version" + command: "java" + args: ["-version"] + expectedError: ['openjdk version "11.*'] + - name: "java installation" + command: "which" + args: ["java"] + expectedOutput: ["/usr/lib/jvm/java-11-openjdk-amd64/bin/java"] + - name: "Hadoop installation" + command: "which" + args: ["hadoop"] + expectedOutput: ["/hadoop/hadoop-2.7.0/bin/hadoop"] + - name: "Beeline installation" + command: "which" + args: ["beeline"] + expectedOutput: ["/apache-hive-1.2.2-bin/bin/beeline"] + - name: check All libs + command: "bash" + args: [ + "-c", + " + Rscript -e 'library(tidyverse)' && + Rscript -e 'library(aws.s3)' && + Rscript -e 'library(arrow)' && + Rscript -e 'library(openxlsx)' && + Rscript -e 'library(data.table)' + Rscript -e 'library(odbc)';", + ] + exitCode: 0 diff --git a/version.properties b/version.properties index 3bafe1de7..2676b1e59 100644 --- a/version.properties +++ b/version.properties @@ -1,6 +1,6 @@ -version.buildmeta= +version.buildmeta=sdktechno-238 version.major=1 version.minor=164 version.patch=0 version.prerelease= -version.semver=1.164.0 +version.semver=1.164.0+sdktechno-238