Sdktechno 238: Add R & Rstudio 4.3 (#591)

* Add R & Rstudio 4.3

technologies/app/rstudio/metadata.yaml

@@ -67,7 +67,7 @@ contexts:
  releaseNotes: First version of RStudio 4.2 into Saagie.
  available: true
  trustLevel: stable
- recommended: true
+ recommended: false
  ports:
  - port: 80
  name: Rstudio
@@ -77,4 +77,20 @@ contexts:
  dockerInfo:
  image: "saagie/rstudio"
  baseTag: "4.2"
- version: "4.2-1.141.0_SDKTECHNO-211"
+ version: "4.2-1.141.0_SDKTECHNO-211"
+ - id: rstudio-4.3
+ label: "4.3"
+ releaseNotes: First version of RStudio 4.3 into Saagie.
+ available: true
+ trustLevel: stable
+ recommended: true
+ ports:
+ - port: 80
+ name: Rstudio
+ rewriteUrl: false
+ basePath: SAAGIE_BASE_PATH
+ volumes: ["/home"]
+ dockerInfo:
+ image: "saagie/rstudio"
+ baseTag: "4.3"
+ version: "4.3-1.163.0_sdktechno-238"

technologies/app/rstudio/rstudio-4.2/context.yaml

@@ -3,7 +3,7 @@ label: "4.2"
 releaseNotes: First version of RStudio 4.2 into Saagie.
 available: true
 trustLevel: stable
-recommended: true
+recommended: false
 ports:
  - port: 80
  name: Rstudio

technologies/app/rstudio/rstudio-4.3/Dockerfile

@@ -0,0 +1,25 @@
+ARG base_img
+
+FROM ${base_img} AS BASE_IMG
+
+# Nginx isntall and remove default conf
+RUN apt-get update -qq && apt-get install -yqq --no-install-recommends nginx \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/ \
+ && rm /etc/nginx/sites-enabled/default
+COPY resources/server.conf /etc/nginx/sites-enabled/rstudio.conf
+COPY resources/userconf /etc/cont-init.d/userconf
+COPY resources/init_rstudio.sh /init_rstudio.sh
+
+# Be sure rstudio user has full access to his home directory
+RUN mkdir -p /home/rstudio && \
+ chown -R rstudio:rstudio /home/rstudio && \
+ chmod -R 755 /home/rstudio && \
+ chmod 500 /init_rstudio.sh
+
+EXPOSE 80
+
+COPY resources/entrypoint.sh /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]
+
+CMD ["/bin/sh", "-c", "/init_rstudio.sh"]

technologies/app/rstudio/rstudio-4.3/build.gradle.kts

@@ -0,0 +1,31 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Copyright 2019-2021.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import com.bmuschko.gradle.docker.DockerRemoteApiPlugin
+import com.saagie.technologies.SaagieTechnologiesGradlePlugin
+
+
+apply<DockerRemoteApiPlugin>()
+apply<SaagieTechnologiesGradlePlugin>()
+
+
+tasks.withType(com.bmuschko.gradle.docker.tasks.image.DockerBuildImage::class) {
+ this.buildArgs.put(
+ "base_img",
+ "saagie/r-image:4.3-1.163.0_sdktechno-238"
+ )
+}

technologies/app/rstudio/rstudio-4.3/context.yaml

@@ -0,0 +1,12 @@
+id: rstudio-4.3
+label: "4.3"
+releaseNotes: First version of RStudio 4.3 into Saagie.
+available: true
+trustLevel: stable
+recommended: true
+ports:
+ - port: 80
+ name: Rstudio
+ rewriteUrl: false
+ basePath: SAAGIE_BASE_PATH
+volumes: ["/home"]

technologies/app/rstudio/rstudio-4.3/dockerInfo.yaml

@@ -0,0 +1,4 @@
+image: saagie/rstudio
+baseTag: 4.3
+dynamicVersion: 1.163.0_sdktechno-238
+version: 4.3-1.163.0_sdktechno-238

technologies/app/rstudio/rstudio-4.3/image_test.yaml

@@ -0,0 +1,96 @@
+schemaVersion: "2.0.0"
+
+metadataTest:
+ env:
+ - key: LANG
+ value: en_US.UTF-8
+ - key: JAVA_HOME
+ value: /usr/lib/jvm/java-11-openjdk-amd64
+ - key: HADOOP_HOME
+ value: /hadoop/hadoop-2.7.0
+ - key: HIVE_HOME
+ value: /apache-hive-1.2.2-bin
+
+fileExistenceTests:
+ - name: "entrypoint"
+ path: "/entrypoint.sh"
+ shouldExist: true
+ permissions: "-rwxr-xr-x"
+ - name: "init_rstudio"
+ path: "/init_rstudio.sh"
+ shouldExist: true
+ permissions: "-r-x------"
+ - name: "rstudio_home"
+ path: "/home/rstudio"
+ shouldExist: true
+ permissions: "drwxr-xr-x"
+ uid: 1000
+ gid: 1000
+ - name: No default nginx conf
+ path: /etc/nginx/sites-enabled/default
+ shouldExist: false
+ - name: rstudio nginx conf
+ path: "/etc/nginx/sites-enabled/rstudio.conf"
+ shouldExist: true
+ permissions: "-rw-r--r--"
+ - name: Impala ODBC dependecy
+ path: "/opt/cloudera/impalaodbc/Setup/odbcinst.ini"
+ shouldExist: true
+
+fileContentTests:
+ - name: "OBBC Impala content"
+ path: "/opt/cloudera/impalaodbc/Setup/odbcinst.ini"
+ expectedContents:
+ [
+ "Cloudera ODBC Driver for Impala 32-bit=Installed",
+ "Cloudera ODBC Driver for Impala 64-bit=Installed",
+ "Driver=/opt/cloudera/impalaodbc/lib/32/libclouderaimpalaodbc32.so",
+ "Driver=/opt/cloudera/impalaodbc/lib/64/libclouderaimpalaodbc64.so"
+ ]
+ - name: "RStudio conf"
+ path: "/etc/nginx/sites-enabled/rstudio.conf"
+ expectedContents:
+ [
+ "listen 80 default_server;",
+ "listen \\[::\\]:80 default_server",
+ "location SAAGIE_BASE_PATH {",
+ "proxy_pass http:\\/\\/localhost:8787;",
+ ]
+
+commandTests:
+ - name: "Workdir"
+ command: "pwd"
+ expectedOutput: ["/"]
+ - name: "R version"
+ command: "R"
+ args: ["--version"]
+ expectedOutput: ['R version 4.3.*']
+ - name: "java version"
+ command: "java"
+ args: ["-version"]
+ expectedError: ['openjdk version "11.*']
+ - name: "java installation"
+ command: "which"
+ args: ["java"]
+ expectedOutput: ["/usr/lib/jvm/java-11-openjdk-amd64/bin/java"]
+ - name: "Hadoop installation"
+ command: "which"
+ args: ["hadoop"]
+ expectedOutput: ["/hadoop/hadoop-2.7.0/bin/hadoop"]
+ - name: "Beeline installation"
+ command: "which"
+ args: ["beeline"]
+ expectedOutput: ["/apache-hive-1.2.2-bin/bin/beeline"]
+ - name: check All libs
+ command: "bash"
+ args: [
+ "-c",
+ "
+ Rscript -e 'library(tidyverse)' &&
+ Rscript -e 'library(aws.s3)' &&
+ Rscript -e 'library(arrow)' &&
+ Rscript -e 'library(openxlsx)' &&
+ Rscript -e 'library(data.table)' &&
+ Rscript -e 'library(odbc)';",
+ ]
+ exitCode: 0

technologies/app/rstudio/rstudio-4.3/resources/entrypoint.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+if [[ -z ${RSTUDIO_ADMIN_PASSWORD} || -z ${RSTUDIO_PASSWORD} ]]; then
+ echo "ERROR : Missing environment variables. In order to work, this app needs the following environment variables set : "
+ echo "RSTUDIO_ADMIN_PASSWORD : Password for the user admin, with root permissions"
+ echo "RSTUDIO_PASSWORD : Password for the user rstudio"
+ exit 1
+fi
+
+R_HOME=$(Rscript -e 'Sys.getenv("R_HOME")' | sed -rn 's/^\[[[:digit:]]+\] "(.*)"/\1/p')
+
+# Change default CRAN to ENV VAR if present
+if [[ -z "${R_CUSTOM_CRAN}" ]]; then
+ echo "No custom CRAN defined, using default, you can configure one using R_CUSTOM_CRAN env var"
+else
+ echo "Custom CRAN used: ${R_CUSTOM_CRAN}"
+ sed -ri "s#(CRAN = ')[^']*#\1${R_CUSTOM_CRAN}#g" "${R_HOME}/etc/Rprofile.site"
+fi
+
+sed -i 's:SAAGIE_BASE_PATH:'"$SAAGIE_BASE_PATH"':g' /etc/nginx/sites-enabled/rstudio.conf
+nginx && /init_rstudio.sh

technologies/app/rstudio/rstudio-4.3/resources/init_rstudio.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+# Grab runtime parameters
+TEMP=`getopt -o p: --long port: -- "$@"`
+eval set -- "$TEMP"
+
+while true ; do
+ case "$1" in
+ -p|--port)
+ case "$2" in
+ "") shift 2 ;;
+ *) RSTUDIO_PORT=$2 ; shift 2 ;;
+ esac ;;
+ --) shift ; break ;;
+ *) echo "Internal error!" ; exit 1 ;;
+ esac
+done
+
+# check if rstudio needs to be run on a custom port
+if [ -z $RSTUDIO_PORT ];
+then
+ echo "INFO: no port given. RStudio will run on default port (8787)."
+ export PORT0=8787
+ export RSTUDIO_PORT=8787
+else
+ # If not already set, set a fake PORT0 variable (used in spark-env.sh)
+ if [ -z $PORT0 ]
+ then
+ export PORT0=$(( $RSTUDIO_PORT+1 ))
+ echo "WARNING: no PORT0 environment variable provided. $PORT0 will be used..."
+ fi
+ echo "www-port=$RSTUDIO_PORT" >> /etc/rstudio/rserver.conf
+fi
+
+echo "Running RStudio on port $RSTUDIO_PORT"
+
+/init &
+
+# wait a bit for initialization to end before modifying the users...
+sleep 5
+
+echo "Init users"
+
+# We need to make rstudio user the owner of his own home directory as it will be owned by root if we mount a docker volume to /home (weird but true...)
+mkdir -p /home/rstudio
+chown -R rstudio:rstudio /home/rstudio
+chmod -R 755 /home/rstudio
+
+# create an admin user who will be able to create new users directly from RStudio IDE
+useradd admin --home /home/admin --create-home -p $(openssl passwd -1 ${RSTUDIO_ADMIN_PASSWORD}) --groups sudo,shadow,rstudio,staff
+
+# Add backupusers script for admin user
+mkdir -p /home/admin
+echo "sudo tar cf /home/admin/users_backup.tar /etc/passwd /etc/shadow 2> /dev/null" > /home/admin/backupusers
+chown admin:admin /home/admin/backupusers
+chmod 700 /home/admin/backupusers
+
+# then restore previous users if necessary
+if [ -e /home/admin/users_backup.tar ]
+then
+ echo "Restoring users..."
+ rm -rf /tmp/userrestore
+ mkdir -p /tmp/userrestore
+
+ tar xf /home/admin/users_backup.tar -C /tmp/userrestore --strip-components=1
+
+ cat /tmp/userrestore/shadow | while read LINE
+ do
+ # echo $LINE
+
+ USER=$(echo $LINE | awk -F':' '{print $1}')
+ ENCRYPTED_PASSWD=$(echo $LINE | awk -F':' '{print $2}')
+
+ if ! grep -q "^$USER:" /etc/passwd
+ then
+ echo "- restoring $USER"
+ HOME_DIR=$(grep "^$USER:" /tmp/userrestore/passwd | awk -F':' '{print $6}')
+ echo " home dir: $HOME_DIR"
+ useradd $USER --home $HOME_DIR --create-home -p $ENCRYPTED_PASSWD
+ chown -R $USER:$USER $HOME_DIR
+ fi
+
+ done
+fi
+
+# propagate environment variables to every user
+R_HOME=$(Rscript -e 'Sys.getenv("R_HOME")' | sed -rn 's/^\[[[:digit:]]+\] "(.*)"/\1/p')
+
+env | while read VAR
+do
+ if ! grep -q "$VAR" /ROOT_ENV_VAR && ! echo "$VAR" | grep -q "RSTUDIO_ADMIN_PASSWORD"
+ then
+ echo $VAR >> $R_HOME/etc/Renviron.site
+ fi
+done
+
+# keep the container running...
+tail -f /dev/null

technologies/app/rstudio/rstudio-4.3/resources/server.conf

@@ -0,0 +1,24 @@
+map $http_connection $upgrade_requested {
+ default upgrade;
+ '' close;
+}
+server {
+ listen 80 default_server;
+ listen [::]:80 default_server;
+ server_name _;
+
+ client_max_body_size 2G;
+
+ location SAAGIE_BASE_PATH {
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection $upgrade_requested;
+ proxy_read_timeout 20d;
+
+ rewrite ^SAAGIE_BASE_PATH/(.*)$ /$1 break;
+ rewrite ^SAAGIE_BASE_PATH$ / break;
+ proxy_pass http://localhost:8787;
+ proxy_redirect http://localhost:8787/ $scheme://$hostSAAGIE_BASE_PATH/;
+ proxy_redirect https://localhost:8787/ $scheme://$hostSAAGIE_BASE_PATH/;
+ }
+}