Merge branch 'feature-config' into release-3.0

etc/config/cert.sh

@@ -0,0 +1,45 @@
+#
+#    Ophidia Server
+#    Copyright (C) 2012-2023 CMCC Foundation
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+#!/bin/bash
+openssl req -newkey rsa:1024 \
+    -passout pass:abcd \
+    -subj "/" -sha1 \
+    -keyout rootkey.pem \
+    -out rootreq.pem
+openssl x509 -req -in rootreq.pem \
+    -passin pass:abcd \
+    -sha1 -extensions v3_ca \
+    -signkey rootkey.pem \
+    -out rootcert.pem
+cat rootcert.pem rootkey.pem  > cacert.pem
+
+openssl req -newkey rsa:1024 \
+    -passout pass:abcd \
+    -subj "/" -sha1 \
+    -keyout serverkey.pem \
+    -out serverreq.pem
+openssl x509 -req \
+    -in serverreq.pem \
+    -passin pass:abcd \
+    -sha1 -extensions usr_cert \
+    -CA cacert.pem  \
+    -CAkey cacert.pem \
+    -CAcreateserial \
+    -out servercert.pem
+cat servercert.pem serverkey.pem rootcert.pem > myserver.pem

etc/config/my.cnf.template

@@ -0,0 +1,42 @@
+[client]
+port    = $PORT
+socket  = $PREFIX/$MYSQLENV/var/run/mysqld/mysqld.sock
+###user = $USER
+###password = $PASSWORD
+
+
+[mysqld_safe]
+log-error       = $PREFIX/$MYSQLENV/var/log/mysql/mysql.log
+pid-file        = $PREFIX/$MYSQLENV/var/run/mysqld/mysqld.pid
+
+[mysqld]
+# Basic Settings
+bind-address = 0.0.0.0
+user            = mysql
+pid-file        = $PREFIX/$MYSQLENV/var/run/mysqld/mysqld.pid
+socket          = $PREFIX/$MYSQLENV/var/run/mysqld/mysqld.sock
+port            = $PORT
+basedir         = $PREFIX/$MYSQLENV
+datadir         = $PREFIX/$MYSQLENV/var/lib/mysql
+tmpdir          = $PREFIX/$MYSQLENV/tmp
+plugin_dir      = $PREFIX/$MYSQLENV/ophidia-primitives/lib
+
+lc-messages-dir         = $PREFIX/$MYSQLENV/share/mysql
+
+
+# CACHES AND LIMITS #
+connect_timeout                 = 60
+max_connections                 = 1024
+max_user_connections    = 0
+max_allowed_packet              = 10M
+open_files_limit                = 4096
+
+# Disabling symbolic-links is recommended to prevent assorted security risks
+symbolic-links  = 0
+
+# Recommended in standard MySQL setup
+sql_mode        = NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES
+
+# LOGGING
+general_log_file        = $PREFIX/$MYSQLENV/var/log/mysql/mysql.log
+log-error           = $PREFIX/$MYSQLENV/var/log/mysql/mysql-error.log

etc/config/mysql.sql.template

@@ -0,0 +1,16 @@
+ALTER USER 'root'@'localhost' IDENTIFIED BY '$PASSWORD';
+CREATE USER 'root'@'%' IDENTIFIED BY '$PASSWORD';
+GRANT ALL privileges ON *.* TO 'root'@'%' WITH GRANT OPTION;
+GRANT SUPER on *.* to 'root'@'%';
+GRANT EXECUTE on mysql.* to 'root'@'%';
+FLUSH PRIVILEGES;
+
+CREATE DATABASE ophidiadb;
+CREATE DATABASE oph_dimensions;
+CREATE USER '$USER'@'%' IDENTIFIED BY '$PASSWORD';
+GRANT ALL PRIVILEGES ON ophidiadb.* TO '$USER'@'%';
+GRANT ALL PRIVILEGES ON oph_dimensions.* TO '$USER'@'%';
+GRANT SUPER on *.* to '$USER'@'%';
+GRANT TRIGGER ON ophidiadb.* TO '$USER'@'%';
+FLUSH PRIVILEGES;
+SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));

etc/config/ophidia-conf.sh

@@ -0,0 +1,152 @@
+#
+#    Ophidia Server
+#    Copyright (C) 2012-2023 CMCC Foundation
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+#!/bin/bash
+set -e
+myhost=`hostname`
+myport=3306
+myuser=ophidia
+mypassword=ophidia2022
+mymemory=16384
+numnodes=1
+export PREFIX=/scratch/shared/
+export MYSQLENV=mysql-env
+export CNFDIR=$(realpath "$(dirname "$0")")
+export TMP=$PREFIX/tmp;
+export SPACK_USER_CACHE_PATH=$PREFIX/spack/tmp;
+sed -i "s|# build_jobs: 16|build_jobs: 2|g" $SPACK_ROOT/etc/spack/defaults/config.yaml
+echo "[LOG] OPHIDIA COMPONENTS INSTALLATION"
+spack install ophidia-primitives
+spack install ophidia-io-server
+spack install ophidia-analytics-framework
+spack install ophidia-server
+export OPHIDIA_PRIMITIVES=`spack location -i ophidia-primitives`
+export OPHIDIA_IOSERVER=`spack location -i ophidia-io-server`
+export OPHIDIA_FRAMEWORK=`spack location -i ophidia-analytics-framework`
+export OPHIDIA_SERVER=`spack location -i ophidia-server`
+echo "[LOG] CONDA ENVIRONMENT CREATION"
+wget https://repo.anaconda.com/archive/Anaconda3-2022.10-Linux-x86_64.sh
+bash Anaconda3-2022.10-Linux-x86_64.sh
+conda create python=3.7 --prefix=$PREFIX/$MYSQLENV -y
+source $CONDA_PREFIX/etc/profile.d/conda.sh
+conda activate $PREFIX/$MYSQLENV
+
+echo "[LOG] PYOPHIDIA INSTALLATION IN THE ENVIRONMENT: $MYSQLENV"
+conda install -c conda-forge -y pyophidia
+
+echo "[LOG] MYSQL INSTALLATION IN THE ENVIRONMENT: $MYSQLENV"
+conda install -y mysql
+
+echo "[LOG] MYSQL CONFIGURATION"
+cd $PREFIX/$MYSQLENV
+mkdir etc
+mkdir -p var/lib/mysql
+mkdir -p var/log/mysql/mysql
+mkdir -p var/run/mysqld
+mkdir -p tmp
+mkdir ophidia-primitives
+cd $PREFIX/$MYSQLENV/ophidia-primitives/
+ln -s $OPHIDIA_PRIMITIVES/lib/ lib
+cp $CNFDIR/my.cnf.template $PREFIX/$MYSQLENV/etc/my.cnf
+sed -i "s|\$PORT|${myport}|g" $PREFIX/$MYSQLENV/etc/my.cnf
+sed -i "s|\$USER|${myuser}|g" $PREFIX/$MYSQLENV/etc/my.cnf
+sed -i "s|\$PASSWORD|${mypassword}|g" $PREFIX/$MYSQLENV/etc/my.cnf
+sed -i "s|\$PREFIX|$PREFIX|g" $PREFIX/$MYSQLENV/etc/my.cnf
+sed -i "s|\$MYSQLENV|$MYSQLENV|g" $PREFIX/$MYSQLENV/etc/my.cnf
+
+echo "[LOG] STARTING MYSQL SERVER..."
+cd $PREFIX/$MYSQLENV
+bin/mysqld --defaults-file=$PREFIX/$MYSQLENV/etc/my.cnf --user=$USER --initialize-insecure
+bin/mysqld_safe --defaults-file=$PREFIX/$MYSQLENV/etc/my.cnf &
+sleep 5
+cp $CNFDIR/mysql.sql.template $CNFDIR/mysql.sql
+sed -i "s|\$USER|${myuser}|g" $CNFDIR/mysql.sql
+sed -i "s|\$PASSWORD|${mypassword}|g" $CNFDIR/mysql.sql
+bin/mysql -u root --skip-password < $CNFDIR/mysql.sql
+sed -i "s|###||g" $PREFIX/$MYSQLENV/etc/my.cnf
+chmod 600 $PREFIX/$MYSQLENV/etc/my.cnf
+
+bin/mysql --defaults-file=$PREFIX/$MYSQLENV/etc/my.cnf ophidiadb < $OPHIDIA_FRAMEWORK/etc/ophidiadb.sql
+spack load gsl
+bin/mysql -u root -p${mypassword} mysql < $OPHIDIA_PRIMITIVES/etc/create_func.sql
+
+echo "[LOG] CHECK SRUN INSTALLATION"
+if ! command -v srun > $CNFDIR/slurm_path.txt
+then
+    if ! spack location -i slurm
+    then
+        $CNFDIR/slurm-conf.sh
+    fi
+    export SLURM_PATH=`spack location -i slurm`
+else
+    SRUN_PATH=$(cat $CNFDIR/slurm_path.txt)
+    tmp=$(dirname $SRUN_PATH)
+    export SLURM_PATH=${tmp%/*}
+fi
+echo "[LOG] SLURM INSTALLED IN $SLURM_PATH"
+echo "[LOG] OPHIDIA SERVER CONFIGURATION"
+sed -i "s|127.0.0.1|${myhost}|g" $OPHIDIA_SERVER/etc/ophidiadb.conf
+sed -i "s|3306|${myport}|g" $OPHIDIA_SERVER/etc/ophidiadb.conf
+sed -i "s|root|${myuser}|g" $OPHIDIA_SERVER/etc/ophidiadb.conf
+sed -i "s|abcd|${mypassword}|g" $OPHIDIA_SERVER/etc/ophidiadb.conf
+sed -i "s|127.0.0.1|${myhost}|g" $OPHIDIA_SERVER/etc/server.conf
+sed -i "s|OPH_EXTRA_LOCATION=/usr/local/ophidia/extra|OPH_EXTRA_LOCATION=${SLURM_PATH}|g" $OPHIDIA_SERVER/etc/server.conf
+sed -i "s|OPH_IOSERVER_LOCATION=/usr/local/ophidia/oph-cluster/oph-io-server|OPH_IOSERVER_LOCATION=${OPHIDIA_IOSERVER}|g" $OPHIDIA_SERVER/etc/server.conf
+echo "ENABLE_CLUSTER_DEPLOYMENT=yes" >> $OPHIDIA_SERVER/etc/server.conf
+echo "ENABLE_CLUSTER_INCREASE=yes" >> $OPHIDIA_SERVER/etc/server.conf
+sed -i "s|127.0.0.1|${myhost}|g" $OPHIDIA_SERVER/etc/script/start_ioserver.sh
+sed -i "s|3306|${myport}|g" $OPHIDIA_SERVER/etc/script/start_ioserver.sh
+sed -i "s|root|${myuser}|g" $OPHIDIA_SERVER/etc/script/start_ioserver.sh
+sed -i "s|abcd|${mypassword}|g" $OPHIDIA_SERVER/etc/script/start_ioserver.sh
+sed -i "s|65001|65000|g" $OPHIDIA_SERVER/etc/script/oph_ioserver.conf.template
+if [ $numnodes -eq 1 ]; then
+        sed -i "s|--exclusive||g" $OPHIDIA_SERVER/etc/script/oph_start.sh
+fi
+
+cd $OPHIDIA_SERVER
+mkdir -p log
+mkdir -p etc/cert
+cd $OPHIDIA_SERVER/etc/cert
+cp $CNFDIR/cert.sh $OPHIDIA_SERVER/etc/cert/cert.sh
+./cert.sh
+spack stage ophidia-server
+spack cd ophidia-server
+
+cp -r authz $OPHIDIA_SERVER
+
+echo "[LOG] OPHIDIA ANALYTICS FRAMEWORK CONFIGURATION"
+sed -i "s|127.0.0.1|${myhost}|g" $OPHIDIA_FRAMEWORK/etc/oph_configuration
+sed -i "s|3306|${myport}|g" $OPHIDIA_FRAMEWORK/etc/oph_configuration
+sed -i "s|root|${myuser}|g" $OPHIDIA_FRAMEWORK/etc/oph_configuration
+sed -i "s|abcd|${mypassword}|g" $OPHIDIA_FRAMEWORK/etc/oph_configuration
+sed -i "s|1024|${mymemory}|g" $OPHIDIA_FRAMEWORK/etc/oph_configuration
+sed -i "s|127.0.0.1|${myhost}|g" $OPHIDIA_FRAMEWORK/etc/oph_soap_configuration
+
+mkdir -p $HOME/.ophidia/
+chmod 700 $HOME/.ophidia/
+echo "[mysql]" >> $HOME/.my.cnf
+echo "user=${myuser}" >> $HOME/.my.cnf
+echo "password=${mypassword}" >> $HOME/.my.cnf
+cd $OPHIDIA_FRAMEWORK
+mkdir -p html/sessions
+mkdir -p log
+
+echo "[LOG] MYSQL SERVER SHUTDOWN"
+cd $PREFIX/$MYSQLENV
+bin/mysqladmin -u root shutdown -pophidia2022
+conda deactivate

etc/config/ophidia-run.sh

@@ -0,0 +1,31 @@
+#
+#    Ophidia Server
+#    Copyright (C) 2012-2023 CMCC Foundation
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+#!/bin/bash
+export SLURM_CONF=$HOME/.ophidia/etc/slurm.conf
+cd `spack location -i munge`
+echo "[LOG] STARTING MUNGE..."
+sbin/munged -S $HOME/.ophidia/var_run_munge/munge.socket.2
+cd `spack location -i slurm`
+echo "[LOG] STARTING SLURM..."
+sbin/slurmd
+sbin/slurmctld
+cd $CONDA_PREFIX
+bin/mysqld_safe --defaults-file=/scratch/shared/mysql-env/etc/my.cnf &
+cd `spack location -i ophidia-server`
+bin/oph_server -d 2>&1 > /dev/null < /dev/null &

etc/config/ophidiadb.conf.template

@@ -0,0 +1,5 @@
+OPHDB_NAME=ophidiadb
+OPHDB_HOST=$HOST
+OPHDB_PORT=$PORT
+OPHDB_LOGIN=$USER
+OPHDB_PWD=$PASSWORD

etc/config/slurm-conf.sh

@@ -0,0 +1,55 @@
+#
+#    Ophidia Server
+#    Copyright (C) 2012-2023 CMCC Foundation
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+#!/bin/bash
+set -e
+export PREFIX=/scratch/shared/
+export CNFDIR=$(realpath "$(dirname "$0")")
+echo "[LOG] MUNGE AND SLURM INSTALLATION"
+spack install slurm
+spack load munge
+spack load slurm
+export MUNGE_DIR=`spack location -i munge`
+export SLURM_DIR=`spack location -i slurm`
+cd $HOME/.ophidia/
+mkdir -p var/log/slurm
+mkdir -p var/run/slurm
+mkdir -p var/spool/slurm
+mkdir -p var/spool/slurmd
+mkdir -p var/log/slurmctld
+mkdir -p etc
+
+echo "[LOG] MUNGE CONFIGURATION"
+cd $MUNGE_DIR
+sbin/mungekey
+mkdir -p var/run
+mkdir -p var/run/munge
+chmod 700 var/log/munge
+chmod 700 etc/munge
+chmod 755 $PREFIX
+ln -s $MUNGE_DIR/var/run/munge $HOME/.ophidia/var_run_munge
+
+echo "[LOG] SLURM CONFIGURATION"
+cd $SLURM_DIR
+cp $CNFDIR/slurm.conf.template $HOME/.ophidia/etc/slurm.conf
+sed -i "s|\$HOME|$HOME|g" $HOME/.ophidia/etc/slurm.conf
+sed -i "s|\$USER|$USER|g" $HOME/.ophidia/etc/slurm.conf
+PROCS=`grep -c ^processor /proc/cpuinfo`
+sed -i "s|\$PROCS|$PROCS|g" $HOME/.ophidia/etc/slurm.conf
+export SLURM_CONF=$HOME/.ophidia/etc/slurm.conf
+echo "[LOG] SLURM CONFIGURED"

etc/config/slurm.conf.template

@@ -0,0 +1,31 @@
+ControlMachine=localhost
+ControlAddr=127.0.0.1
+AuthInfo=socket=$HOME/.ophidia/var_run_munge/munge.socket.2
+AuthType=auth/munge
+CryptoType=crypto/munge
+MpiDefault=none
+ProctrackType=proctrack/pgid
+ReturnToService=1
+SlurmctldPidFile=$HOME/.ophidia/var/run/slurm/slurmctld.pid
+SlurmdPidFile=$HOME/.ophidia/var/run/slurm/slurmd.pid
+SlurmdSpoolDir=$HOME/.ophidia/var/spool/slurmd
+SlurmUser=$USER
+SlurmdUser=$USER
+StateSaveLocation=$HOME/.ophidia/var/spool/slurmd
+SwitchType=switch/none
+TaskPlugin=task/none
+SchedulerType=sched/backfill
+#SelectType=select/linear
+SelectType=select/cons_res
+SelectTypeParameters=CR_CPU
+#FastSchedule=1
+AccountingStorageType=accounting_storage/none
+ClusterName=cluster
+JobCompType=jobcomp/none
+JobAcctGatherType=jobacct_gather/none
+SlurmctldDebug=3
+SlurmctldLogFile=$HOME/.ophidia/var/log/slurmctld/slurmctld.log
+SlurmdDebug=3
+SlurmdLogFile=$HOME/.ophidia/var/log/slurm/slurmd.log
+NodeName=localhost NodeAddr=127.0.0.1 State=UNKNOWN Procs=$PROCS
+PartitionName=main Nodes=localhost Default=YES MaxTime=INFINITE State=UP