Merge branch 'master' into issue-1312

* master:
  Patch for move to Pause after dialing an invalid client This refer to #2358
  Added comment for restcomm.xml configuration option This refer to #2346
  add section about officially supported language
  Print log messages for S3AccessTool if waiting/uploading the file takes more than X sec (configurable) This refer to #2346
  Patch for On Dial invalid/unregistered client, VoiceInterpreter should either execute Dial Action or move the the next verb This refer #2358
  Patch for On Dial invalid/unregistered client, VoiceInterpreter should either execute Dial Action or move the the next verb This close #2358
  Simulate long operation of FileUtils.waitFor putting the thread to sleep for 6000 ms This refer to #2346
  Update S3AccessTool for integration with S3 Mock service. This refer to #2346
  Simulate long operation of FileUtils.waitFor putting the thread to sleep for 6000 ms This refer to #2346
  Patch to provide custom executor for long/blocking tasks Patch to add FileUtils.waitFor on S3AccessTool This refer #2339 This refer #2324
  Simulate long operation of FileUtils.waitFor putting the thread to sleep for 1 minutes This refer to #2346
  Added deveveloper doc for #2340
  Minor tweak in logging statement #2340
  Fixed the MySQL command to save privileges changes
  Chenged references to mariadb when mysql applies..
  Patch to properly prepare S3Client for integration with S3Ninja for testing This close #2346
  Add license scan status and report
  Added config-based RVD/rcmlserver resolver
  Patch to provide custom executor for long/blocking tasks Patch to remove blocking on S3AccessTool This close #2339 This close #2324
  Configure 'rvdUrl' in dashboard.json from advanced.conf/RVD_URL

README.md

@@ -1,5 +1,6 @@
 RestComm
 ========
+[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bhttps%3A%2F%2Fgithub.com%2FRestComm%2FRestcomm-Connect.svg?type=shield)](https://app.fossa.io/projects/git%2Bhttps%3A%2F%2Fgithub.com%2FRestComm%2FRestcomm-Connect?ref=badge_shield)
 
 The Open Source Cloud Communications Platform
 
@@ -47,6 +48,8 @@ RestComm is lead by [TeleStax](http://www.telestax.com/), Inc. and developed col
 
 RestComm is licensed under dual license policy. The default license is the Free Open Source GNU Affero GPL v3.0. Alternatively a commercial license can be obtained from Telestax ([contact form](http://www.telestax.com/contactus/#InquiryForm))
 
+[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bhttps%3A%2F%2Fgithub.com%2FRestComm%2FRestcomm-Connect.svg?type=large)](https://app.fossa.io/projects/git%2Bhttps%3A%2F%2Fgithub.com%2FRestComm%2FRestcomm-Connect?ref=badge_large)
+
 Continuous Integration and Delivery
 ========
 [![RestComm Continuous Job](http://www.cloudbees.com/sites/default/files/Button-Built-on-CB-1.png)](https://mobicents.ci.cloudbees.com/job/RestComm/)

restcomm/configuration/config-scripts/as7-config-scripts/restcomm/autoconfig.d/config-dashboard.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+##
+## Configures dashboard.json based on global configuration options in restcomm.conf and advanced.conf.
+##
+## requirements:
+##
+##	RESTCOMM_HOME env variable to be set
+##  dashboard.json should be in place (under $RESTCOMM_HOME/standalone/deployments/restcomm-management.war)
+##
+## Author: otsakir@gmail.com - Orestis Tsakiridis
+
+echo "Configuring Dashboard..."
+
+
+# MAIN
+if [ -z "$RESTCOMM_HOME" ]
+then
+    echo "RESTCOMM_HOME env variable not set. Aborting."
+    exit 1
+fi
+
+
+# Variables
+DASHBOARD_ROOT="$RESTCOMM_HOME"/standalone/deployments/restcomm-management.war
+DASHBOARD_JSON_FILE="$DASHBOARD_ROOT"/conf/dashboard.json
+
+sed -i  "s|\"rvdUrl\":\"[^\"]*\"|\"rvdUrl\":\"$RVD_URL/restcomm-rvd\"|" "$DASHBOARD_JSON_FILE"
+
+echo "Dasboard configured:"
+cat $DASHBOARD_JSON_FILE

restcomm/restcomm.application/src/main/java/org/restcomm/connect/application/Bootstrapper.java

@@ -29,6 +29,7 @@
 import org.restcomm.connect.mscontrol.api.MediaServerInfo;
 import org.restcomm.connect.mscontrol.jsr309.Jsr309ControllerFactory;
 import org.restcomm.connect.mscontrol.mms.MmsControllerFactory;
+import scala.concurrent.ExecutionContext;
 
 import javax.media.mscontrol.MsControlException;
 import javax.media.mscontrol.MsControlFactory;
@@ -56,6 +57,7 @@ public final class Bootstrapper extends SipServlet implements SipServletListener
     private static final Logger logger = Logger.getLogger(Bootstrapper.class);
 
     private ActorSystem system;
+    private ExecutionContext ec;
 
     public Bootstrapper() {
         super();
@@ -212,10 +214,10 @@ private String home(final ServletContext context) {
         }
     }
 
-    private DaoManager storage(final Configuration configuration, Configuration daoManagerConfiguration, final ClassLoader loader) throws ObjectInstantiationException {
+    private DaoManager storage(final Configuration configuration, Configuration daoManagerConfiguration, final ClassLoader loader, final ExecutionContext ec) throws ObjectInstantiationException {
         final String classpath = daoManagerConfiguration.getString("dao-manager[@class]");
         final DaoManager daoManager = (DaoManager) new ObjectFactory(loader).getObjectInstance(classpath);
-        daoManager.configure(configuration, daoManagerConfiguration );
+        daoManager.configure(configuration, daoManagerConfiguration, ec );
         daoManager.start();
         return daoManager;
     }
@@ -286,10 +288,11 @@ public void servletInitialized(SipServletContextEvent event) {
             system = ActorSystem.create("RestComm", settings, loader);
             // Share the actor system with other servlets.
             context.setAttribute(ActorSystem.class.getName(), system);
+            ec = system.dispatchers().lookup("restcomm-blocking-dispatcher");
             // Create the storage system.
             DaoManager storage = null;
             try {
-                storage = storage(xml, daoManagerConf, loader);
+                storage = storage(xml, daoManagerConf, loader, ec);
             } catch (final ObjectInstantiationException exception) {
                 logger.error("ObjectInstantiationException during initialization: ", exception);
             }

restcomm/restcomm.application/src/main/resources/application.conf

@@ -1,21 +1,21 @@
 akka {
- # Event handlers to register at boot time (Logging$DefaultLogger logs to STDOUT)
-event-handlers = ["akka.event.slf4j.Slf4jEventHandler"]
+  # Event handlers to register at boot time (Logging$DefaultLogger logs to STDOUT)
+  event-handlers = ["akka.event.slf4j.Slf4jEventHandler"]
 
-# Log level used by the configured loggers (see "event-handlers") as soon
-# as they have been started; before that, see "stdout-loglevel"
-# Options: OFF, ERROR, WARNING, INFO, DEBUG
-loglevel = "INFO"
+  # Log level used by the configured loggers (see "event-handlers") as soon
+  # as they have been started; before that, see "stdout-loglevel"
+  # Options: OFF, ERROR, WARNING, INFO, DEBUG
+  loglevel = "INFO"
 
-# Log level for the very basic logger activated during AkkaApplication startup
-# Options: OFF, ERROR, WARNING, INFO, DEBUG
-stdout-loglevel = "INFO"
+  # Log level for the very basic logger activated during AkkaApplication startup
+  # Options: OFF, ERROR, WARNING, INFO, DEBUG
+  stdout-loglevel = "INFO"
 
-# Log the complete configuration at INFO level when the actor system is started.
-# This is useful when you are uncertain of what configuration is used.
-log-config-on-start = off
+  # Log the complete configuration at INFO level when the actor system is started.
+  # This is useful when you are uncertain of what configuration is used.
+  log-config-on-start = off
 
-# Configuration reference http://doc.akka.io/docs/akka/2.1.4/general/configuration.html
+  # Configuration reference http://doc.akka.io/docs/akka/2.1.4/general/configuration.html
 
   actor {
 
@@ -28,7 +28,14 @@ log-config-on-start = off
     # Default value 20s is too high
     # creation-timeout = 20s
     creation-timeout = 10s
-
-    }
- }
-
+  }
+}
+
+restcomm-blocking-dispatcher {
+  type = Dispatcher
+  executor = "thread-pool-executor"
+  thread-pool-executor {
+    fixed-pool-size = 32
+  }
+  throughput = 1
+}

restcomm/restcomm.application/src/main/webapp/WEB-INF/conf/restcomm.xml

@@ -41,6 +41,9 @@
 		<recordings-path>file://${restcomm:home}/recordings</recordings-path>
 		<recordings-uri>/restcomm/recordings</recordings-uri>
 
+		<!-- if S3AccessTool takes more than the duration here, then Restcomm will report this -->
+		<recording-max-delay>2000</recording-max-delay>
+
 		<!-- The URL to the errors dictionary. -->
 		<error-dictionary-uri>/restcomm/errors</error-dictionary-uri>
 

restcomm/restcomm.commons/src/main/java/org/restcomm/connect/commons/Configurable.java

@@ -20,10 +20,11 @@
 package org.restcomm.connect.commons;
 
 import org.apache.commons.configuration.Configuration;
+import scala.concurrent.ExecutionContext;
 
 /**
  * @author quintana.thomas@gmail.com (Thomas Quintana)
  */
 public interface Configurable {
-    void configure(Configuration configuration, Configuration daoManagerConfiguration);
+    void configure(Configuration configuration, Configuration daoManagerConfiguration, ExecutionContext ec);
 }

restcomm/restcomm.commons/src/main/java/org/restcomm/connect/commons/amazonS3/S3AccessTool.java

@@ -29,12 +29,15 @@
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3Client;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.S3ClientOptions;
 import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
 import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.PutObjectRequest;
 import com.amazonaws.services.s3.model.StorageClass;
 import org.apache.commons.io.FileUtils;
 import org.apache.log4j.Logger;
+import org.joda.time.DateTime;
+import org.restcomm.connect.commons.configuration.RestcommConfiguration;
 
 import javax.activation.MimetypesFileTypeMap;
 import java.io.File;
@@ -63,6 +66,7 @@ public class S3AccessTool {
     private boolean testing;
     private String testingUrl;
     private AmazonS3 s3client;
+    private int maxDelay;
 
     public S3AccessTool(final String accessKey, final String securityKey, final String bucketName, final String folder,
             final boolean reducedRedundancy, final int minutesToRetainPublicUrl, final boolean removeOriginalFile,
@@ -77,6 +81,7 @@ public S3AccessTool(final String accessKey, final String securityKey, final Stri
         this.bucketRegion = bucketRegion;
         this.testing = testing;
         this.testingUrl = testingUrl;
+        this.maxDelay = RestcommConfiguration.getInstance().getMain().getRecordingMaxDelay();
     }
 
     public AmazonS3 getS3client() {
@@ -86,6 +91,7 @@ public AmazonS3 getS3client() {
             s3client = new AmazonS3Client(awsCreds);
             s3client.setRegion(Region.getRegion(Regions.fromName(bucketRegion)));
             s3client.setEndpoint(testingUrl);
+            s3client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).disableChunkedEncoding().build());
         } else {
             s3client = AmazonS3ClientBuilder.standard().withRegion(Regions.fromName(bucketRegion))
                     .withCredentials(new AWSStaticCredentialsProvider(awsCreds)).build();
@@ -94,15 +100,17 @@ public AmazonS3 getS3client() {
         return s3client;
     }
 
-    public URI uploadFile(final String fileToUpload) {
+    public boolean uploadFile(final String fileToUpload) {
         if (s3client == null) {
             s3client = getS3client();
         }
         if(logger.isInfoEnabled()){
             logger.info("S3 Region: "+bucketRegion.toString());
         }
         try {
-            if (testing && (!testingUrl.isEmpty() || !testingUrl.equals(""))) {
+            URI fileUri = URI.create(fileToUpload);
+            File file = new File(fileUri);
+            if (!file.exists() && testing && (!testingUrl.isEmpty() || !testingUrl.equals(""))) {
 //                s3client.setEndpoint(testingUrl);
 //                s3client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
                 FileUtils.touch(new File(URI.create(fileToUpload)));
@@ -111,14 +119,33 @@ public URI uploadFile(final String fileToUpload) {
             bucket.append(bucketName);
             if (folder != null && !folder.isEmpty())
                 bucket.append("/").append(folder);
-            URI fileUri = URI.create(fileToUpload);
             if (logger.isInfoEnabled()) {
                 logger.info("File to upload to S3: " + fileUri.toString());
             }
-            File file = new File(fileUri);
 
+            DateTime start = DateTime.now();
             while (!FileUtils.waitFor(file, 30)){}
+            DateTime end = DateTime.now();
+            double waitDuration = (end.getMillis() - start.getMillis())/1000;
+            if (waitDuration > maxDelay || testing) {
+                if (logger.isInfoEnabled()) {
+                    String msg = String.format("Finished waiting for recording file %s. Wait time %,.2f. Size of file in bytes %s", fileUri.toString(), waitDuration, file.length());
+                    logger.info(msg);
+                }
+            }
             if (file.exists()) {
+                start = DateTime.now();
+                if (testing) {
+                    try {
+                        if (logger.isInfoEnabled()) {
+                            logger.info("Will make thread sleep for 1 minute simulating the long operation of FileUtils.waitFor");
+                        }
+                        Thread.sleep(6000);
+                    } catch (Exception e) {
+                        logger.error("Exception while sleepig simulating the long operation waiting for the file");
+
+                    }
+                }
                 PutObjectRequest putRequest = new PutObjectRequest(bucket.toString(), file.getName(), file);
                 ObjectMetadata metadata = new ObjectMetadata();
                 metadata.setContentType(new MimetypesFileTypeMap().getContentType(file));
@@ -130,12 +157,18 @@ public URI uploadFile(final String fileToUpload) {
                 if (removeOriginalFile) {
                     removeLocalFile(file);
                 }
-                URI recordingS3Uri = s3client.getUrl(bucket.toString(), file.getName()).toURI();
-                return recordingS3Uri;
-//                return downloadUrl.toURI();
+                end = DateTime.now();
+                waitDuration = (end.getMillis() - start.getMillis())/1000;
+                if (waitDuration > maxDelay || testing) {
+                    if (logger.isInfoEnabled()) {
+                        String msg = String.format("File %s uploaded to S3 successfully. Upload time %,.2f sec", fileUri.toString(), waitDuration);
+                        logger.info(msg);
+                    }
+                }
+                return true;
             } else {
                 logger.error("Timeout waiting for the recording file: "+file.getAbsolutePath());
-                return null;
+                return false;
             }
          } catch (AmazonServiceException ase) {
             logger.error("Caught an AmazonServiceException");
@@ -144,18 +177,37 @@ public URI uploadFile(final String fileToUpload) {
             logger.error("AWS Error Code:   " + ase.getErrorCode());
             logger.error("Error Type:       " + ase.getErrorType());
             logger.error("Request ID:       " + ase.getRequestId());
-            return null;
+            return false;
         } catch (AmazonClientException ace) {
             logger.error("Caught an AmazonClientException ");
             logger.error("Error Message: " + ace.getMessage());
-            return null;
-        } catch (URISyntaxException e) {
-            logger.error("URISyntaxException: "+e.getMessage());
-            return null;
+            return false;
         } catch (IOException e) {
             logger.error("Problem while trying to touch recording file for testing", e);
-            return null;
+            return false;
+        }
+    }
+
+    public URI getS3Uri(final String fileToUpload) {
+        if (s3client == null) {
+            s3client = getS3client();
+        }
+        StringBuffer bucket = new StringBuffer();
+        bucket.append(bucketName);
+        if (folder != null && !folder.isEmpty())
+            bucket.append("/").append(folder);
+        URI fileUri = URI.create(fileToUpload);
+        File file = new File(fileUri);
+        URI recordingS3Uri = null;
+        try {
+            recordingS3Uri = s3client.getUrl(bucketName, file.getName()).toURI();
+        } catch (URISyntaxException e) {
+            logger.error("Problem during creation of S3 URI");
+        }
+        if (logger.isInfoEnabled()) {
+            logger.info("Created S3Uri for file: " + fileUri.toString());
         }
+        return recordingS3Uri;
     }
 
     public URI getPublicUrl (String fileName) throws URISyntaxException {
@@ -190,4 +242,4 @@ private void removeLocalFile(final File file) {
             }
         }
     }
-}
+}

restcomm/restcomm.commons/src/main/java/org/restcomm/connect/commons/configuration/sets/MainConfigurationSet.java

@@ -41,4 +41,6 @@ public interface MainConfigurationSet {
     String getInstanceId();
 
     String getApiVersion();
+
+    int getRecordingMaxDelay ();
 }

restcomm/restcomm.commons/src/main/java/org/restcomm/connect/commons/configuration/sets/impl/MainConfigurationSetImpl.java

@@ -51,6 +51,7 @@ public class MainConfigurationSetImpl extends ConfigurationSet implements MainCo
     private String hostname;
     private String instanceId;
     private String apiVersion;
+    private int recordingMaxDelay;
 
     public static final String BYPASS_LB_FOR_CLIENTS = "bypass-lb-for-clients";
     private boolean bypassLbForClients = false;
@@ -94,6 +95,8 @@ public MainConfigurationSetImpl(ConfigurationSource source) {
         this.hostname = resolveRelativeUrlHostname;
         bypassLbForClients = bypassLb;
         apiVersion = source.getProperty("runtime-settings.api-version");
+
+        this.recordingMaxDelay = Integer.parseInt(source.getProperty("runtime-setting.recording-max-delay", "2000"));
     }
 
     public MainConfigurationSetImpl(SslMode sslMode, int responseTimeout, boolean useHostnameToResolveRelativeUrls, String hostname, String instanceId, boolean bypassLbForClients) {
@@ -160,4 +163,9 @@ public void setBypassLbForClients(boolean bypassLbForClients) {
     public String getApiVersion() {
         return apiVersion;
     }
+
+    @Override
+    public int getRecordingMaxDelay () {
+        return recordingMaxDelay;
+    }
 }

restcomm/restcomm.commons/src/main/java/org/restcomm/connect/commons/configuration/sources/ApacheConfigurationSource.java

@@ -41,4 +41,9 @@ public String getProperty(String key) {
         return apacheConfiguration.getString(key);
     }
 
+    @Override
+    public String getProperty (String key, String defValue) {
+        return apacheConfiguration.getString(key, defValue);
+    }
+
 }

restcomm/restcomm.commons/src/main/java/org/restcomm/connect/commons/configuration/sources/ConfigurationSource.java

@@ -26,4 +26,5 @@
  */
 public interface ConfigurationSource {
     String getProperty(String key);
+    String getProperty (String key, String defValue);
 }