Merge pull request #4626 from EC528-Dataverse-Scaling/4040-glassfish1

Adding Glassfish Statefulsets for OpenShift Deployment #4617

conf/openshift/openshift.json

@@ -124,15 +124,18 @@
       }
     },
     {
-      "kind": "DeploymentConfig",
-      "apiVersion": "v1",
+      "kind": "StatefulSet",
+      "apiVersion": "apps/v1beta1",
       "metadata": {
         "name": "dataverse-glassfish",
         "annotations": {
-          "template.alpha.openshift.io/wait-for-ready": "true"
+          "template.alpha.openshift.io/wait-for-ready": "true",
+          "alpha.image.policy.openshift.io/resolve-names": "*"
         }
       },
       "spec": {
+        "serviceName" : "dataverse-glassfish",
+        "replicas": 1, 
         "template": {
           "metadata": {
             "labels": {
@@ -143,14 +146,26 @@
             "containers": [
               {
                 "name": "dataverse-plus-glassfish",
-                "image": "dataverse-plus-glassfish",
+                "image": "iqss/dataverse-glassfish:latest",
                 "ports": [
                   {
                     "containerPort": 8080,
                     "protocol": "TCP"
                   }
                 ],
+                "resources": {"limits": {"memory": "1024Mi"
+                  }
+
+                  },
                 "env": [
+
+                       { "name": "MY_POD_NAME",
+                          "valueFrom": {
+                          "fieldRef": {
+                           "fieldPath": "metadata.name"
+                         }
+                        }
+                       },
                   {
                     "name": "POSTGRES_SERVER",
                     "value": "dataverse-postgresql-0"
@@ -184,7 +199,7 @@
                     "value": "dvndb"
                   }
                 ],
-                "imagePullPolicy": "IfNotPresent",
+                "imagePullPolicy": "Always",
                 "securityContext": {
                   "capabilities": {},
                   "privileged": false
@@ -200,7 +215,9 @@
             "intervalSeconds": 1,
             "timeoutSeconds": 300
           },
-          "resources": {}
+          "resources": {"limits": {
+                    "memory": "512Mi"
+                  }}
         },
         "triggers": [
           {
@@ -220,9 +237,11 @@
             "type": "ConfigChange"
           }
         ],
-        "replicas": 1,
         "selector": {
-          "name": "iqss-dataverse-glassfish"
+          "name": "iqss-dataverse-glassfish",
+         "matchLabels" : {
+           "name": "iqss-dataverse-glassfish"
+        }
         }
       }
     },
@@ -252,6 +271,7 @@
                 "command": [
                 "sh", "-c", "echo 'Setting up Postgres Master/Slave replication...'; [[ `hostname` =~ -([0-9]+)$ ]] || exit 1; ordinal=${BASH_REMATCH[1]}; if [[ $ordinal -eq 0 ]]; then run-postgresql-master; else run-postgresql-slave; fi;"
                 ],
+
                 "ports": [
                   {
                     "containerPort": 5432,
@@ -306,7 +326,7 @@
                     "memory": "256Mi"
                   }
                 },
-                "imagePullPolicy": "IfNotPresent",
+                "imagePullPolicy": "Always",
                 "securityContext": {
                   "capabilities": {},
                   "privileged": false
@@ -371,7 +391,7 @@
             "containers": [
               {
                 "name": "iqss-dataverse-solr",
-                "image": "iqss-dataverse-solr",
+                "image": "iqss-dataverse-solr:latest",
                 "ports": [
                   {
                     "containerPort": 8983,
@@ -380,10 +400,10 @@
                 ],
                 "resources": {
                   "limits": {
-                    "memory": "256Mi"
+                    "memory": "1024Mi"
                   }
                 },
-                "imagePullPolicy": "IfNotPresent",
+                "imagePullPolicy": "Always",
                 "securityContext": {
                   "capabilities": {},
                   "privileged": false

doc/sphinx-guides/source/developers/containers.rst

@@ -30,7 +30,7 @@ At this point, you might want to consider going through the Minishift quickstart
 Start Minishift
 ~~~~~~~~~~~~~~~
 
-``minishift start --vm-driver=virtualbox --memory=4GB``
+``minishift start --vm-driver=virtualbox --memory=8GB``
 
 Make the OpenShift Client Binary (oc) Executable
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -150,6 +150,21 @@ If you are interested in changing the OpenShift config file for Dataverse at ``c
 
 The slower way to iterate on the ``openshift.json`` file is to delete the project and re-create it.
 
+Scaling Dataverse by Increasing Replicas in a StatefulSet
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The Glassfish and PostgreSQL Pods are in a "StatefulSet" which is a concept from OpenShift and Kubernetes that you can read about at https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/
+
+As of this writing, the ``openshift.json`` file has a single "replica" for each of these two stateful sets. It's possible to increase the number of replicas from 1 to 3, for example, with this command:
+
+``oc scale statefulset/dataverse-glassfish --replicas=3``
+
+The command above should result in two additional Glassfish pods being spun up. The name of the pods is significant and there is special logic in the "zeroth" pod ("dataverse-glassfish-0" and "dataverse-postgresql-0"). For example, only "dataverse-glassfish-0" makes itself the dedicated timer server as explained in :doc:`/admin/timers` section of the Admin Guide. "dataverse-glassfish-1" and other higher number pods will not be configured as a timer server.
+
+Once you have multiple Glassfish servers you may notice bugs that will require additional configuration to fix. One such bug has to do with Dataverse logos which are stored at ``/usr/local/glassfish4/glassfish/domains/domain1/docroot/logos`` on each of the Glassfish servers. This means that the logo will look fine when you just uploaded it because you're on the server with the logo on the local file system but when you visit that dataverse in the future and you're on a differernt Glassfish server, you will see a broken image. (You can find some discussion of this logo bug at https://github.com/IQSS/dataverse-aws/issues/10 and http://irclog.iq.harvard.edu/dataverse/2016-10-21 .) This is all "advanced" installation territory (see the :doc:`/installation/advanced` section of the Installation Guide) and OpenShift might be a good environment in which to work on some of these bugs.
+
+Multiple PostgreSQL servers are possible within the OpenShift environment as well and have been set up with some amount of replication. "dataverse-postgresql-0" is the master and non-zero pods are the slaves. We have just scratched the surface of this configuration but replication from master to slave seems to we working. Future work could include failover and making Dataverse smarter about utilizing multiple PostgreSQL servers for reads. Right now we assume Dataverse is only being used with a single PostgreSQL server and that it's the master.
+
 Running Containers to Run as Root in Minishift
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

doc/sphinx-guides/source/developers/windows.rst

@@ -113,7 +113,7 @@ From Git Bash, run the following commands:
 Start Minishift VM and Run Dataverse
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-``minishift start --vm-driver=virtualbox --memory=4GB``
+``minishift start --vm-driver=virtualbox --memory=8GB``
 
 ``eval $(minishift oc-env)``
 

scripts/installer/glassfish-setup.sh

@@ -191,7 +191,20 @@ fi
 
 ###
 # Set up the data source for the timers
-./asadmin $ASADMIN_OPTS set configs.config.server-config.ejb-container.ejb-timer-service.timer-datasource=jdbc/VDCNetDS
+
+if [ -z "$MY_POD_NAME" ]
+ then
+	./asadmin $ASADMIN_OPTS set configs.config.server-config.ejb-container.ejb-timer-service.timer-datasource=jdbc/VDCNetDS
+
+ else
+    echo $MY_POD_NAME
+    if [ $MY_POD_NAME == "dataverse-glassfish-0" ]
+      then
+    	./asadmin $ASADMIN_OPTS set configs.config.server-config.ejb-container.ejb-timer-service.timer-datasource=jdbc/VDCNetDS
+        echo "Only I Run The Jobs"
+      fi
+  fi
+
 
 ###
 # Add the necessary JVM options: 
@@ -216,7 +229,20 @@ fi
 ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddoi.username=apitest"
 ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddoi.baseurlstring=https\://ezid.cdlib.org"
 # "I am the timer server" option:
-./asadmin $ASADMIN_OPTS create-jvm-options "-Ddataverse.timerServer=true"
+
+if [ -z "$MY_POD_NAME" ]
+  then
+	./asadmin $ASADMIN_OPTS create-jvm-options "-Ddataverse.timerServer=true"
+
+   else
+       if [ $MY_POD_NAME == "dataverse-glassfish-0" ]
+          then
+              ./asadmin $ASADMIN_OPTS create-jvm-options "-Ddataverse.timerServer=true"
+
+    fi
+ fi
+
+
 
 # enable comet support
 ./asadmin $ASADMIN_OPTS set server-config.network-config.protocols.protocol.http-listener-1.http.comet-support-enabled="true"