Configure JPA L2 cache coordination between instances on Openshift® / Kubernetes

assembly/assembly-wsmaster-war/pom.xml

@@ -324,6 +324,10 @@
             <groupId>org.eclipse.persistence</groupId>
             <artifactId>org.eclipse.persistence.core</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.eclipse.persistence</groupId>
+            <artifactId>org.eclipse.persistence.extension</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.eclipse.persistence</groupId>
             <artifactId>org.eclipse.persistence.jpa</artifactId>
@@ -340,6 +344,14 @@
             <groupId>org.flywaydb</groupId>
             <artifactId>flyway-core</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.jgroups</groupId>
+            <artifactId>jgroups</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.jgroups.kubernetes</groupId>
+            <artifactId>kubernetes</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.postgresql</groupId>
             <artifactId>postgresql</artifactId>

assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/WsMasterModule.java

@@ -83,6 +83,7 @@
 import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesInfrastructure;
 import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftInfraModule;
 import org.eclipse.che.workspace.infrastructure.openshift.OpenShiftInfrastructure;
+import org.eclipse.persistence.config.CacheCoordinationProtocol;
 import org.eclipse.persistence.config.PersistenceUnitProperties;
 import org.flywaydb.core.internal.util.PlaceholderReplacer;
 
@@ -214,15 +215,23 @@ protected void configure() {
     persistenceProperties.put(PersistenceUnitProperties.LOGGING_LEVEL, "SEVERE");
     persistenceProperties.put(
         PersistenceUnitProperties.NON_JTA_DATASOURCE, "java:/comp/env/jdbc/che");
-
     bindConstant().annotatedWith(Names.named("jndi.datasource.name")).to("java:/comp/env/jdbc/che");
 
+    String infrastructure = System.getenv("CHE_INFRASTRUCTURE_ACTIVE");
     if (Boolean.valueOf(System.getenv("CHE_MULTIUSER"))) {
       persistenceProperties.put(
           PersistenceUnitProperties.EXCEPTION_HANDLER_CLASS,
           "org.eclipse.che.core.db.postgresql.jpa.eclipselink.PostgreSqlExceptionHandler");
 
-      configureMultiUserMode();
+      if (OpenShiftInfrastructure.NAME.equals(infrastructure)
+          || KubernetesInfrastructure.NAME.equals(infrastructure)) {
+        persistenceProperties.put(
+            PersistenceUnitProperties.COORDINATION_PROTOCOL, CacheCoordinationProtocol.JGROUPS);
+        persistenceProperties.put(
+            PersistenceUnitProperties.COORDINATION_JGROUPS_CONFIG, "jgroups/che-tcp.xml");
+      }
+
+      configureMultiUserMode(persistenceProperties, infrastructure);
     } else {
       persistenceProperties.put(
           PersistenceUnitProperties.EXCEPTION_HANDLER_CLASS,
@@ -234,7 +243,6 @@ protected void configure() {
         new com.google.inject.persist.jpa.JpaPersistModule("main")
             .properties(persistenceProperties));
 
-    String infrastructure = System.getenv("CHE_INFRASTRUCTURE_ACTIVE");
     if (OpenShiftInfrastructure.NAME.equals(infrastructure)) {
       install(new OpenShiftInfraModule());
     } else if (KubernetesInfrastructure.NAME.equals(infrastructure)) {
@@ -267,7 +275,8 @@ private void configureSingleUserMode() {
     bind(org.eclipse.che.security.oauth.OAuthAuthenticationService.class);
   }
 
-  private void configureMultiUserMode() {
+  private void configureMultiUserMode(
+      Map<String, String> persistenceProperties, String infrastructure) {
     bind(TemplateProcessor.class).to(STTemplateProcessorImpl.class);
     bind(DataSource.class).toProvider(org.eclipse.che.core.db.JndiDataSourceProvider.class);
 

assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/jgroups/che-tcp.xml

@@ -0,0 +1,52 @@
+<!--
+
+    Copyright (c) 2012-2018 Red Hat, Inc.
+    All rights reserved. This program and the accompanying materials
+    are made available under the terms of the Eclipse Public License v1.0
+    which accompanies this distribution, and is available at
+    http://www.eclipse.org/legal/epl-v10.html
+
+    Contributors:
+      Red Hat, Inc. - initial API and implementation
+
+-->
+<config xmlns="urn:org:jgroups"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="urn:org:jgroups http://www.jgroups.org/schema/JGroups-3.1.xsd">
+  <TCP
+    bind_addr="match-interface:eth0"
+    bind_port="${TCP_PORT:7800}"
+    recv_buf_size="5M"
+    send_buf_size="1M"
+    max_bundle_size="64K"
+    enable_diagnostics="true"
+    thread_naming_pattern="cl"
+
+    thread_pool.min_threads="0"
+    thread_pool.max_threads="500"
+    thread_pool.keep_alive_time="30000" />
+
+  <kubernetes.KUBE_PING
+    labels="app=che" />
+
+  <MERGE2 min_interval="10000"
+    max_interval="30000"/>
+  <FD_SOCK/>
+  <FD timeout="3000" max_tries="3"/>
+  <VERIFY_SUSPECT timeout="1500"/>
+  <BARRIER/>
+  <pbcast.NAKACK2 use_mcast_xmit="false"
+    discard_delivered_msgs="true"/>
+  <UNICAST/>
+  <pbcast.STABLE stability_delay="1000" desired_avg_gossip="50000"
+    max_bytes="4M"/>
+  <pbcast.GMS print_local_addr="true" join_timeout="3000"
+
+    view_bundling="true"/>
+  <UFC max_credits="2M"
+    min_threshold="0.4"/>
+  <MFC max_credits="2M"
+    min_threshold="0.4"/>
+  <FRAG2 frag_size="60K"/>
+  <pbcast.STATE_TRANSFER/>
+</config>

deploy/kubernetes/helm/che/templates/deployment.yaml

@@ -218,6 +218,10 @@ spec:
             configMapKeyRef:
               key: CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON
               name: che
+        - name: OPENSHIFT_KUBE_PING_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
         - name: CHE_INFRA_KUBERNETES_SERVER__STRATEGY
           valueFrom:
             configMapKeyRef:
@@ -243,6 +247,8 @@ spec:
           name: http
         - containerPort: 8000
           name: http-debug
+        - containerPort: 8888
+          name: jgroups-ping
         readinessProbe:
           httpGet:
             path: /api/system/state

deploy/kubernetes/kubectl/che-kubernetes.yaml

@@ -249,6 +249,10 @@ items:
               configMapKeyRef:
                 key: CHE_INFRA_KUBERNETES_INGRESS_ANNOTATIONS__JSON
                 name: che
+          - name: OPENSHIFT_KUBE_PING_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
           - name: CHE_LOGS_APPENDERS_IMPL
             valueFrom:
               configMapKeyRef:
@@ -269,6 +273,8 @@ items:
             name: http
           - containerPort: 8000
             name: http-debug
+          - containerPort: 8888
+            name: jgroups-ping
           readinessProbe:
             httpGet:
               path: /api/system/state

deploy/openshift/che-openshift.yml

@@ -37,6 +37,10 @@ items:
       spec:
         containers:
         - env:
+          - name: OPENSHIFT_KUBE_PING_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
           #CHE_MASTER_CONFIG
           image: registry.devshift.net/che/che:291ebd9-fabric8-e470151
           imagePullPolicy: IfNotPresent
@@ -53,6 +57,8 @@ items:
             name: http
           - containerPort: 8000
             name: http-debug
+          - containerPort: 8888
+            name: jgroups-ping
           readinessProbe:
             httpGet:
               path: /api/system/state