From a0afaef2e21c1e238be82860ae83f7ddce9f0c54 Mon Sep 17 00:00:00 2001
From: "Joseph P. White" <jpwhite4@buffalo.edu>
Date: Tue, 27 Nov 2018 11:27:13 -0500
Subject: [PATCH 1/2] Update Job Viewer API

---
 classes/DataWarehouse/Query/Query.php         |  41 +++++
 classes/DataWarehouse/Query/iJobMetadata.php  | 143 ++++++++++++++++++
 .../WarehouseControllerProvider.php           |  18 +--
 configuration/portal_settings.ini             |   1 -
 html/gui/js/modules/job_viewer/SearchPanel.js |   7 +-
 html/index.php                                |   2 +-
 templates/portal_settings.template            |   1 -
 7 files changed, 196 insertions(+), 17 deletions(-)
 create mode 100644 classes/DataWarehouse/Query/iJobMetadata.php

diff --git a/classes/DataWarehouse/Query/Query.php b/classes/DataWarehouse/Query/Query.php
index 9a284cc39b..2b6b504508 100644
--- a/classes/DataWarehouse/Query/Query.php
+++ b/classes/DataWarehouse/Query/Query.php
@@ -462,6 +462,31 @@ public function getStatFields()
         return $this->_stat_fields;
     }
 
+    /**
+     * Add a where condition to the query and add the data to the pdo parameters. This
+     * function should be used when the right hand side of the where condition is untrused
+     * user supplied data.
+     *
+     * Note this function does not handle pdo parameterization of 'IN' conditions.
+     */
+    public function addPdoWhereCondition(\DataWarehouse\Query\Model\WhereCondition $where_condition)
+    {
+        // key on the non-parameterized form since the substitution string is different every time.
+        $key = $where_condition->__toString();
+
+        if (isset($this->_where_conditions[$key])) {
+            return;
+        }
+
+        $namedParam = $this->getNamedParameterMarker($where_condition->_right);
+
+        $this->_where_conditions[$key] = new \DataWarehouse\Query\Model\WhereCondition(
+            $where_condition->_left,
+            $where_condition->_operation,
+            $namedParam
+        );
+    }
+
     public function addWhereCondition(\DataWarehouse\Query\Model\WhereCondition $where_condition)
     {
         $this->_where_conditions[$where_condition->__toString()] = $where_condition;
@@ -725,6 +750,22 @@ public function getCountQueryString()
         $data_query .= ") as a WHERE a.total IS NOT NULL";
         return $data_query;
     }
+
+    /**
+     * Store a bound parameter for the query and return the named parameter
+     * marker that should be used in the SQL query.
+     *
+     * @param the value to bind to the query
+     * @return string a named parameter marker.
+     */
+    protected function getNamedParameterMarker($value)
+    {
+        $pdosubst = ':subst' . $this->pdoindex;
+        $this->pdoparams[$pdosubst] = $value;
+        $this->pdoindex += 1;
+        return $pdosubst;
+    }
+
     public function setParameters(array $parameters = array())
     {
         $this->parameters = $parameters;
diff --git a/classes/DataWarehouse/Query/iJobMetadata.php b/classes/DataWarehouse/Query/iJobMetadata.php
new file mode 100644
index 0000000000..41e52e2672
--- /dev/null
+++ b/classes/DataWarehouse/Query/iJobMetadata.php
@@ -0,0 +1,143 @@
+<?php
+/**
+ * Interface defining the JobMetadata API. The JobMetadata API is used to provide the information
+ * about a given job (or tracked entity) that is available to XDMoD but not stored in the datawarehouse
+ * aggregate tables. See the JobDataset API for the access functions to retrieve information about
+ * a job (or tracked entity) from the datawarehouse itself.
+ *
+ * All modules that implement realms that show information in the Job Viewer
+ * tab must provide an implementation of this interface.
+ */
+
+namespace DataWarehouse\Query;
+
+interface iJobMetadata
+{
+    /**
+     * Return information about all of the data available for a given job. The supported
+     * data categories are defined in \DataWarehouse\Query\RawQueryTypes
+     *
+     * @param XDUser $user The authenticated user.
+     * @param $jobid The unique identifier for the job in the datawarehouse fact tables.
+     *
+     * @returns array() of \DataWarehouse\Query\RawQueryTypes listing all of
+     * the data types available for the requested job. or null if the job does
+     * not exist.
+     */
+    public function getJobMetadata(\XDUser $user, $jobid);
+
+    /**
+     * Return information that will be displayed in the detailed metrics tab of the Job
+     * Viewer. This function will be called if the job has the DETAILED_METRICS data type.
+     *
+     * @param XDUser $user The authenticated user.
+     * @param $jobid The unique identifier for the job in the datawarehouse fact tables.
+     *
+     * @returns array() of metrics. Each metric should contain the summary statistics for the metric as well as the documentation, data type and units. Metrics may be nested. For example if a job provided a CPU User metric
+     * then this call would return a php array with the following structure:
+     *
+     *  "cpu": {
+     *      "user": {
+     *          "std": 0.0066031459102555,
+     *          "med": 0.0045116925711787,
+     *          "skw": 0.79910187474516,
+     *          "cov": 0.73753588228371,
+     *          "max": 0.021206549192241,
+     *          "min": 0.0033731199203248,
+     *          "cnt": 16,
+     *          "krt": -0.85522002996351,
+     *          "avg": 0.0089529825854839,
+     *          "documentation": "The CPU usage in user mode of the cores that were assigned to the job. This metric reports the overall usage of each core that the job was assigned rather than, for example, the CPU usage of the job processes themselves.",
+     *          "type": "instant",
+     *          "unit": "ratio"
+     *      }
+     *  }
+     */
+    public function getJobSummary(\XDUser $user, $jobid);
+
+    /**
+     * Return information that will be displayed in the executable info tab of the Job Viewer
+     * This function will be called if the job has the EXECUTABLE data type.
+     *
+     * @param XDUser $user The authenticated user.
+     * @param $jobid The unique identifier for the job in the datawarehouse fact tables.
+     *
+     * @returns array() of text with the information about the executable for the job.
+     */
+    public function getJobExecutableInfo(\XDUser $user, $jobid);
+
+    /**
+     * Return metadata about what timeseries information is available for the provided job.
+     * This function will be called if the job has the TIMESERIES_METRICS data type.
+     *
+     * @param XDUser $user The authenticated user.
+     * @param $jobid The unique identifier for the job in the datawarehouse fact tables.
+     *
+     * @returns array() containing timeseries metrics metadata. For example:
+     *
+     *  { "tsid": "cpu_user", "name": "CPU Usage", "leaf": false }
+     *
+     *  where tsid is an (internal) timeseries metric identifier. name is the string that will
+     *  be displayed on the gui and leaf is whether the entry is a leaf node in the metric tree.
+     */
+    public function getJobTimeseriesMetaData(\XDUser $user, $jobid);
+
+    /**
+     * Return metadata about what timeseries information is available for the provided metric for a job.
+     * This function will be called if the job has the TIMESERIES_METRICS data type.
+     *
+     * @param XDUser $user The authenticated user.
+     * @param $jobid The unique identifier for the job in the datawarehouse fact tables.
+     * @param $tsid  The identifier for the timeseries metric. This will be the tsid value returned from a previous
+     * call to getJobTimeseriesMetaData().
+     *
+     * @returns array() containing timeseries metric metadata. for a given metric. For example:
+     *
+     * [{ "nodeid": "node0", "name": "Compute Node 0", "leaf": false },
+     *  { "nodeid": "node1", "name": "Compute Node 1", "leaf": false }]
+     *
+     *  where nodeid is an (internal) timeseries metric identifier. name is the string that will
+     *  be displayed on the gui and leaf is whether the entry is a leaf node in the metric tree.
+     */
+    public function getJobTimeseriesMetricMeta(\XDUser $user, $jobid, $tsid);
+
+    /**
+     * Return metadata about what timeseries information is available for the provided metric for a job.
+     * This function will be called if the job has the TIMESERIES_METRICS data type.
+     *
+     * @param XDUser $user The authenticated user.
+     * @param $jobid The unique identifier for the job in the datawarehouse fact tables.
+     * @param $tsid The name of the metric. This will be the tsid value returned from a previous
+     *                call to getJobTimeseriesMetricMeta().
+     * @param $nodeid The name of the metric. This will be the tsid value returned from a previous
+     *                call to getJobTimeseriesMetricMeta()
+     *
+     * @returns array() containing timeseries metric metadata. for a given metric. For example:
+     *
+     * [{"cpuid": "cpu0", "text": "CPU0", "leaf": true },
+     *  {"cpuid": "cpu1", "text": "CPU1", "leaf": true },
+     *  {"cpuid": "cpu2", "text": "CPU2", "leaf": true },
+     *  {"cpuid": "cpu3", "text": "CPU3", "leaf": true }]
+     *
+     *  where cpuid is an (internal) timeseries metric identifier. name is the string that will
+     *  be displayed on the gui and leaf is whether the entry is a leaf node in the metric tree.
+     */
+    public function getJobTimeseriesMetricNodeMeta(\XDUser $user, $jobid, $tsid, $nodeid);
+
+    /**
+     * Return timeseries data for the provided job. The data available is
+     * obtained via the getJobTimeseriesMetaData(),
+     * getJobTimeseriesMetricMeta() and getJobTimeseriesMetricNodeMeta() if the
+     * data returned from any of these functions has leaf: true, then the corresponding call
+     * to getJobTimeseriesData() will return timeseries data.
+     *
+     * @param XDUser $user The authenticated user.
+     * @param $jobid The unique identifier for the job in the datawarehouse fact tables.
+     * @param $tsid  The timeseires metric identifier.
+     * @param $nodeid The timeseries node identifier.
+     * @param $cpuid  The timeseries cpu identifier.
+     *
+     * @returns array() timeseries data
+     */
+    public function getJobTimeseriesData(\XDUser $user, $jobid, $tsid, $nodeid, $cpuid);
+}
diff --git a/classes/Rest/Controllers/WarehouseControllerProvider.php b/classes/Rest/Controllers/WarehouseControllerProvider.php
index 364c99f889..d1f5a13e69 100644
--- a/classes/Rest/Controllers/WarehouseControllerProvider.php
+++ b/classes/Rest/Controllers/WarehouseControllerProvider.php
@@ -1489,12 +1489,8 @@ private function getJobDataSet(XDUser $user, $realm, $jobId, $action)
             throw new \DataWarehouse\Query\Exceptions\AccessDeniedException;
         }
 
-        $params = array(
-            new \DataWarehouse\Query\Model\Parameter($rawstats['realms'][$realm]['primary_key'], '=', $jobId)
-        );
-
         $QueryClass = "\\DataWarehouse\\Query\\$realm\\JobDataset";
-        $query = new $QueryClass($params, $action);
+        $query = new $QueryClass(array('primary_key' => $jobId), $action);
 
         $allRoles = $user->getAllRoles();
         $query->setMultipleRoleParameters($allRoles, $user);
@@ -2086,15 +2082,17 @@ private function getJobByPrimaryKey(Application $app, \XDUser $user, $realm, $se
             throw new \DataWarehouse\Query\Exceptions\AccessDeniedException;
         }
 
-        if (isset($searchparams['jobref'])) {
+        if (isset($searchparams['jobref']) && is_int($searchparams['jobref'])) {
             $params = array(
-                new \DataWarehouse\Query\Model\Parameter($rawstats['realms'][$realm]['primary_key'], '=', $searchparams['jobref'])
+                'primary_key' => $searchparams['jobref']
             );
-        } else {
+        } elseif (isset($searchparams['resource_id']) && isset($searchparams['local_job_id'])) {
             $params = array(
-                new \DataWarehouse\Query\Model\Parameter("resource_id", "=", $searchparams['resource_id']),
-                new \DataWarehouse\Query\Model\Parameter($rawstats['realms'][$realm]['ident_key'], "=", $searchparams['local_job_id'])
+                'resource_id' => $searchparams['resource_id'],
+                'job_identifier' => $searchparams['local_job_id']
             );
+        } else {
+            throw new BadRequestException('invalid search parameters');
         }
 
         $QueryClass = "\\DataWarehouse\\Query\\$realm\\JobDataset";
diff --git a/configuration/portal_settings.ini b/configuration/portal_settings.ini
index 7ddfc9a153..74c39d4fad 100644
--- a/configuration/portal_settings.ini
+++ b/configuration/portal_settings.ini
@@ -38,7 +38,6 @@ email_token_expiration = "600"
 [features]
 xsede = "off"
 appkernels = "off"
-singlejobviewer = "off"
 
 ; Enable support for multiple service providers.
 ;
diff --git a/html/gui/js/modules/job_viewer/SearchPanel.js b/html/gui/js/modules/job_viewer/SearchPanel.js
index 4c7922bc08..ce0267ea13 100644
--- a/html/gui/js/modules/job_viewer/SearchPanel.js
+++ b/html/gui/js/modules/job_viewer/SearchPanel.js
@@ -645,7 +645,7 @@ XDMoD.Module.JobViewer.SearchPanel = Ext.extend(Ext.Panel, {
             fields: [
                 {name: 'dtype', mapping: 'dtype', type: 'string'},
                 {name: 'jobid', mapping: 'jobid', type: 'int'},
-                {name: 'local_job_id', mapping: 'local_job_id', type: 'int'},
+                { name: 'local_job_id', mapping: 'local_job_id', type: 'string' },
                 {name: 'name', mapping: 'name', type: 'string'},
                 {name: 'realm', mapping: 'realm', type: 'string'},
                 {name: 'resource', mapping: 'resource', type: 'string'},
@@ -979,12 +979,11 @@ XDMoD.Module.JobViewer.SearchPanel = Ext.extend(Ext.Panel, {
                         }
                     },
                     {
-                        xtype: 'numberfield',
+                        xtype: 'textfield',
                         fieldLabel: 'Job Number',
                         emptyText: 'Enter Job #',
                         id: 'basic-localjobid',
-                        allowNegative: false,
-                        autoStripChars: true,
+                        stripCharsRe: /(^\s+|\s+$)/g,
                         width: 200,
                         enableKeyEvents: true,
                         listeners: {
diff --git a/html/index.php b/html/index.php
index a6b665fa1e..1ebde4abf4 100644
--- a/html/index.php
+++ b/html/index.php
@@ -475,7 +475,7 @@ function isReferrer($referrer)
         <script type="text/javascript" src="gui/js/ChangeStack.js"></script>
 
     <?php /* Single Job Viewer */ ?>
-    <?php if (xd_utilities\getConfiguration('features', 'singlejobviewer') == 'on'): ?>
+    <?php if (!empty($rawDataRealms)): ?>
         <script type="text/javascript" src="gui/js/modules/job_viewer/JobViewer.js"></script>
         <script type="text/javascript" src="gui/js/modules/job_viewer/ChartPanel.js"></script>
         <script type="text/javascript" src="gui/js/modules/job_viewer/ChartTab.js"></script>
diff --git a/templates/portal_settings.template b/templates/portal_settings.template
index db22224007..2590f05659 100644
--- a/templates/portal_settings.template
+++ b/templates/portal_settings.template
@@ -38,7 +38,6 @@ email_token_expiration = "[:general_email_token_expiration:]"
 [features]
 xsede = "off"
 appkernels = "off"
-singlejobviewer = "off"
 
 ; Enable support for multiple service providers.
 ;

From 1759ab4455b966b90292cd1694150de5f7808ed4 Mon Sep 17 00:00:00 2001
From: "Joseph P. White" <jpwhite4@buffalo.edu>
Date: Thu, 29 Nov 2018 10:20:13 -0500
Subject: [PATCH 2/2] Fix security vulnerabities

---
 docs/Gemfile.lock        | 219 ------------------------------------
 etl/js/etl.js            |  32 ++++--
 etl/js/package-lock.json | 234 ---------------------------------------
 etl/js/package.json      |   3 +-
 4 files changed, 22 insertions(+), 466 deletions(-)
 delete mode 100644 docs/Gemfile.lock
 delete mode 100644 etl/js/package-lock.json

diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock
deleted file mode 100644
index 10019dadb7..0000000000
--- a/docs/Gemfile.lock
+++ /dev/null
@@ -1,219 +0,0 @@
-GEM
-  remote: https://rubygems.org/
-  specs:
-    activesupport (4.2.8)
-      i18n (~> 0.7)
-      minitest (~> 5.1)
-      thread_safe (~> 0.3, >= 0.3.4)
-      tzinfo (~> 1.1)
-    addressable (2.5.1)
-      public_suffix (~> 2.0, >= 2.0.2)
-    coffee-script (2.4.1)
-      coffee-script-source
-      execjs
-    coffee-script-source (1.12.2)
-    colorator (1.1.0)
-    ethon (0.10.1)
-      ffi (>= 1.3.0)
-    execjs (2.7.0)
-    faraday (0.13.0)
-      multipart-post (>= 1.2, < 3)
-    ffi (1.9.18)
-    forwardable-extended (2.6.0)
-    gemoji (3.0.0)
-    github-pages (152)
-      activesupport (= 4.2.8)
-      github-pages-health-check (= 1.3.5)
-      jekyll (= 3.5.1)
-      jekyll-avatar (= 0.4.2)
-      jekyll-coffeescript (= 1.0.1)
-      jekyll-default-layout (= 0.1.4)
-      jekyll-feed (= 0.9.2)
-      jekyll-gist (= 1.4.1)
-      jekyll-github-metadata (= 2.7.0)
-      jekyll-mentions (= 1.2.0)
-      jekyll-optional-front-matter (= 0.2.0)
-      jekyll-paginate (= 1.1.0)
-      jekyll-readme-index (= 0.1.0)
-      jekyll-redirect-from (= 0.12.1)
-      jekyll-relative-links (= 0.4.1)
-      jekyll-sass-converter (= 1.5.0)
-      jekyll-seo-tag (= 2.2.3)
-      jekyll-sitemap (= 1.0.0)
-      jekyll-swiss (= 0.4.0)
-      jekyll-theme-architect (= 0.1.0)
-      jekyll-theme-cayman (= 0.1.0)
-      jekyll-theme-dinky (= 0.1.0)
-      jekyll-theme-hacker (= 0.1.0)
-      jekyll-theme-leap-day (= 0.1.0)
-      jekyll-theme-merlot (= 0.1.0)
-      jekyll-theme-midnight (= 0.1.0)
-      jekyll-theme-minimal (= 0.1.0)
-      jekyll-theme-modernist (= 0.1.0)
-      jekyll-theme-primer (= 0.4.0)
-      jekyll-theme-slate (= 0.1.0)
-      jekyll-theme-tactile (= 0.1.0)
-      jekyll-theme-time-machine (= 0.1.0)
-      jekyll-titles-from-headings (= 0.4.0)
-      jemoji (= 0.8.0)
-      kramdown (= 1.13.2)
-      liquid (= 4.0.0)
-      listen (= 3.0.6)
-      mercenary (~> 0.3)
-      minima (= 2.1.1)
-      rouge (= 1.11.1)
-      terminal-table (~> 1.4)
-    github-pages-health-check (1.3.5)
-      addressable (~> 2.3)
-      net-dns (~> 0.8)
-      octokit (~> 4.0)
-      public_suffix (~> 2.0)
-      typhoeus (~> 0.7)
-    html-pipeline (2.6.0)
-      activesupport (>= 2)
-      nokogiri (>= 1.4)
-    i18n (0.8.6)
-    jekyll (3.5.1)
-      addressable (~> 2.4)
-      colorator (~> 1.0)
-      jekyll-sass-converter (~> 1.0)
-      jekyll-watch (~> 1.1)
-      kramdown (~> 1.3)
-      liquid (~> 4.0)
-      mercenary (~> 0.3.3)
-      pathutil (~> 0.9)
-      rouge (~> 1.7)
-      safe_yaml (~> 1.0)
-    jekyll-avatar (0.4.2)
-      jekyll (~> 3.0)
-    jekyll-coffeescript (1.0.1)
-      coffee-script (~> 2.2)
-    jekyll-default-layout (0.1.4)
-      jekyll (~> 3.0)
-    jekyll-feed (0.9.2)
-      jekyll (~> 3.3)
-    jekyll-gist (1.4.1)
-      octokit (~> 4.2)
-    jekyll-github-metadata (2.7.0)
-      jekyll (~> 3.1)
-      octokit (~> 4.0, != 4.4.0)
-    jekyll-mentions (1.2.0)
-      activesupport (~> 4.0)
-      html-pipeline (~> 2.3)
-      jekyll (~> 3.0)
-    jekyll-optional-front-matter (0.2.0)
-      jekyll (~> 3.0)
-    jekyll-paginate (1.1.0)
-    jekyll-readme-index (0.1.0)
-      jekyll (~> 3.0)
-    jekyll-redirect-from (0.12.1)
-      jekyll (~> 3.3)
-    jekyll-relative-links (0.4.1)
-      jekyll (~> 3.3)
-    jekyll-sass-converter (1.5.0)
-      sass (~> 3.4)
-    jekyll-seo-tag (2.2.3)
-      jekyll (~> 3.3)
-    jekyll-sitemap (1.0.0)
-      jekyll (~> 3.3)
-    jekyll-swiss (0.4.0)
-    jekyll-theme-architect (0.1.0)
-      jekyll (~> 3.5)
-      jekyll-seo-tag (~> 2.0)
-    jekyll-theme-cayman (0.1.0)
-      jekyll (~> 3.5)
-      jekyll-seo-tag (~> 2.0)
-    jekyll-theme-dinky (0.1.0)
-      jekyll (~> 3.5)
-      jekyll-seo-tag (~> 2.0)
-    jekyll-theme-hacker (0.1.0)
-      jekyll (~> 3.5)
-      jekyll-seo-tag (~> 2.0)
-    jekyll-theme-leap-day (0.1.0)
-      jekyll (~> 3.5)
-      jekyll-seo-tag (~> 2.0)
-    jekyll-theme-merlot (0.1.0)
-      jekyll (~> 3.5)
-      jekyll-seo-tag (~> 2.0)
-    jekyll-theme-midnight (0.1.0)
-      jekyll (~> 3.5)
-      jekyll-seo-tag (~> 2.0)
-    jekyll-theme-minimal (0.1.0)
-      jekyll (~> 3.5)
-      jekyll-seo-tag (~> 2.0)
-    jekyll-theme-modernist (0.1.0)
-      jekyll (~> 3.5)
-      jekyll-seo-tag (~> 2.0)
-    jekyll-theme-primer (0.4.0)
-      jekyll (~> 3.5)
-      jekyll-seo-tag (~> 2.2)
-    jekyll-theme-slate (0.1.0)
-      jekyll (~> 3.5)
-      jekyll-seo-tag (~> 2.0)
-    jekyll-theme-tactile (0.1.0)
-      jekyll (~> 3.5)
-      jekyll-seo-tag (~> 2.0)
-    jekyll-theme-time-machine (0.1.0)
-      jekyll (~> 3.5)
-      jekyll-seo-tag (~> 2.0)
-    jekyll-titles-from-headings (0.4.0)
-      jekyll (~> 3.3)
-    jekyll-watch (1.5.0)
-      listen (~> 3.0, < 3.1)
-    jemoji (0.8.0)
-      activesupport (~> 4.0)
-      gemoji (~> 3.0)
-      html-pipeline (~> 2.2)
-      jekyll (>= 3.0)
-    kramdown (1.13.2)
-    liquid (4.0.0)
-    listen (3.0.6)
-      rb-fsevent (>= 0.9.3)
-      rb-inotify (>= 0.9.7)
-    mercenary (0.3.6)
-    mini_portile2 (2.2.0)
-    minima (2.1.1)
-      jekyll (~> 3.3)
-    minitest (5.10.3)
-    multipart-post (2.0.0)
-    net-dns (0.8.0)
-    nokogiri (1.8.0)
-      mini_portile2 (~> 2.2.0)
-    octokit (4.7.0)
-      sawyer (~> 0.8.0, >= 0.5.3)
-    pathutil (0.14.0)
-      forwardable-extended (~> 2.6)
-    public_suffix (2.0.5)
-    rb-fsevent (0.10.2)
-    rb-inotify (0.9.10)
-      ffi (>= 0.5.0, < 2)
-    rouge (1.11.1)
-    safe_yaml (1.0.4)
-    sass (3.5.1)
-      sass-listen (~> 4.0.0)
-    sass-listen (4.0.0)
-      rb-fsevent (~> 0.9, >= 0.9.4)
-      rb-inotify (~> 0.9, >= 0.9.7)
-    sawyer (0.8.1)
-      addressable (>= 2.3.5, < 2.6)
-      faraday (~> 0.8, < 1.0)
-    terminal-table (1.8.0)
-      unicode-display_width (~> 1.1, >= 1.1.1)
-    thread_safe (0.3.6)
-    typhoeus (0.8.0)
-      ethon (>= 0.8.0)
-    tzinfo (1.2.3)
-      thread_safe (~> 0.1)
-    unicode-display_width (1.3.0)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  github-pages
-
-RUBY VERSION
-   ruby 2.4.1p111
-
-BUNDLED WITH
-   1.15.3
diff --git a/etl/js/etl.js b/etl/js/etl.js
index 5a9740caa3..669bd6f3af 100644
--- a/etl/js/etl.js
+++ b/etl/js/etl.js
@@ -13,14 +13,13 @@
  */
 var version = "0.9beta",
 	events = require('events'),
-	winston = require('winston'),
 	config = require('./config.js'),
 	ETLProfile = require('./lib/etl_profile.js'),
 	util = require('util'),
 	markProcessedRecords = true,
 	DatasetProcessor = require('./lib/dataset_processor.js');
 
-require('winston-mysql-transport');
+var winston = require('winston');
 
 //ETL details
 
@@ -36,15 +35,26 @@ require('winston-mysql-transport');
 var ETL = module.exports = function() {
 	events.EventEmitter.call(this);
 
-	var etlLogger = new winston.Logger({
-		transports: [
-			new winston.transports.Console({
-				colorize: process.stdout.isTTY,
-				timestamp: true,
-				level: process.env.logLevel || "warn"
-			})
-		]
-	});
+    const logFormat = winston.format.combine(
+        winston.format.timestamp(),
+        winston.format.printf(info => {
+            return `${info.timestamp} - ${info.level}: ${info.message}`;
+        })
+    );
+
+    const colourLogFormat = winston.format.combine(
+        winston.format.colorize(),
+        logFormat
+    );
+
+    var etlLogger = winston.createLogger({
+        transports: [
+            new winston.transports.Console({
+                format: process.stdout.isTTY ? colourLogFormat : logFormat,
+                level: process.env.logLevel || 'warn'
+            })
+        ]
+    });
 
 	this.on('message', function(msg) {
 		etlLogger.info(msg);
diff --git a/etl/js/package-lock.json b/etl/js/package-lock.json
deleted file mode 100644
index c1f895754e..0000000000
--- a/etl/js/package-lock.json
+++ /dev/null
@@ -1,234 +0,0 @@
-{
-    "name": "xdmod-etl",
-    "version": "0.0.0",
-    "lockfileVersion": 1,
-    "dependencies": {
-        "async": {
-            "version": "0.2.10",
-            "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz",
-            "integrity": "sha1-trvgsGdLnXGXCMo43owjfLUmw9E="
-        },
-        "aws-sign": {
-            "version": "0.2.0",
-            "resolved": "https://registry.npmjs.org/aws-sign/-/aws-sign-0.2.0.tgz",
-            "integrity": "sha1-xVAThWyBlOyFSgy+yQqrWgTOOsU="
-        },
-        "bignumber.js": {
-            "version": "1.3.0",
-            "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-1.3.0.tgz",
-            "integrity": "sha1-B749FxI1oqt4wO+uUjONZFx2tzU="
-        },
-        "boom": {
-            "version": "0.3.8",
-            "resolved": "https://registry.npmjs.org/boom/-/boom-0.3.8.tgz",
-            "integrity": "sha1-yM2wQUNZEnQWKMBE7Mcy0dF8Ceo="
-        },
-        "bson": {
-            "version": "0.4.23",
-            "resolved": "https://registry.npmjs.org/bson/-/bson-0.4.23.tgz",
-            "integrity": "sha1-5louPHUH/63kEJvHV1p25Q+NqRU="
-        },
-        "cloneextend": {
-            "version": "0.0.3",
-            "resolved": "https://registry.npmjs.org/cloneextend/-/cloneextend-0.0.3.tgz",
-            "integrity": "sha1-3wYpykhFMWr9iEzxthoM30sSn3k="
-        },
-        "colors": {
-            "version": "0.6.2",
-            "resolved": "https://registry.npmjs.org/colors/-/colors-0.6.2.tgz",
-            "integrity": "sha1-JCP+ZnisDF2uiFLl0OW+CMmXq8w="
-        },
-        "combined-stream": {
-            "version": "0.0.7",
-            "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz",
-            "integrity": "sha1-ATfmV7qlp1QcV6w3rF/AfXO03B8="
-        },
-        "cookie-jar": {
-            "version": "0.2.0",
-            "resolved": "https://registry.npmjs.org/cookie-jar/-/cookie-jar-0.2.0.tgz",
-            "integrity": "sha1-ZOzAasl423leS1KQy+SLo3gUAPo="
-        },
-        "core-util-is": {
-            "version": "1.0.2",
-            "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
-            "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
-        },
-        "cryptiles": {
-            "version": "0.1.3",
-            "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-0.1.3.tgz",
-            "integrity": "sha1-GlVnNPBtJLo0hirpy55wmjr7/xw="
-        },
-        "cycle": {
-            "version": "1.0.3",
-            "resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz",
-            "integrity": "sha1-IegLK+hYD5i0aPN5QwZisEbDStI="
-        },
-        "delayed-stream": {
-            "version": "0.0.5",
-            "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz",
-            "integrity": "sha1-1LH0OpPoKW3+AmlPRoC8N6MTxz8="
-        },
-        "es6-promise": {
-            "version": "3.0.2",
-            "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.0.2.tgz",
-            "integrity": "sha1-AQ1YWEI6XxGJeWZfRkhqlcbuK7Y="
-        },
-        "eyes": {
-            "version": "0.1.8",
-            "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz",
-            "integrity": "sha1-Ys8SAjTGg3hdkCNIqADvPgzCC8A="
-        },
-        "forever-agent": {
-            "version": "0.2.0",
-            "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.2.0.tgz",
-            "integrity": "sha1-4cJcetROCcOPIzh2x2/MJP+EOx8="
-        },
-        "form-data": {
-            "version": "0.0.10",
-            "resolved": "https://registry.npmjs.org/form-data/-/form-data-0.0.10.tgz",
-            "integrity": "sha1-2zRaU3jYau6x7V1VO4aawZLS9e0="
-        },
-        "hawk": {
-            "version": "0.10.2",
-            "resolved": "https://registry.npmjs.org/hawk/-/hawk-0.10.2.tgz",
-            "integrity": "sha1-mzYd7pWpMWQObVBOBWCaj8OsRdI="
-        },
-        "hoek": {
-            "version": "0.7.6",
-            "resolved": "https://registry.npmjs.org/hoek/-/hoek-0.7.6.tgz",
-            "integrity": "sha1-YPvZBFV1Qc0rh5Wr8wihs3cOFVo="
-        },
-        "inherits": {
-            "version": "2.0.3",
-            "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
-            "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
-        },
-        "ini": {
-            "version": "1.2.1",
-            "resolved": "https://registry.npmjs.org/ini/-/ini-1.2.1.tgz",
-            "integrity": "sha1-f3dOLyJ1LNHay/nGMyPfKhZOvKM="
-        },
-        "isarray": {
-            "version": "0.0.1",
-            "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
-            "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
-        },
-        "json-stringify-safe": {
-            "version": "3.0.0",
-            "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-3.0.0.tgz",
-            "integrity": "sha1-nbew5TDH8onF6MhDKvGRwv91pbM="
-        },
-        "mime": {
-            "version": "1.2.11",
-            "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz",
-            "integrity": "sha1-WCA+7Ybjpe8XrtK32evUfwpg3RA="
-        },
-        "mongodb": {
-            "version": "2.1.21",
-            "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-2.1.21.tgz",
-            "integrity": "sha1-dkcJ28zrWCW06zH5U5X5Zf1EInI="
-        },
-        "mongodb-core": {
-            "version": "1.3.21",
-            "resolved": "https://registry.npmjs.org/mongodb-core/-/mongodb-core-1.3.21.tgz",
-            "integrity": "sha1-/hKee+4rOybBQJ3gKrYNA/YpHMo="
-        },
-        "mysql": {
-            "version": "2.2.0",
-            "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.2.0.tgz",
-            "integrity": "sha1-d5nNAtdPkyZDTXNReRyUaYFIW5g=",
-            "dependencies": {
-                "readable-stream": {
-                    "version": "1.1.14",
-                    "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
-                    "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk="
-                }
-            }
-        },
-        "node-uuid": {
-            "version": "1.4.8",
-            "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.8.tgz",
-            "integrity": "sha1-sEDrCSOWivq/jTL7HxfxFn/auQc="
-        },
-        "oauth-sign": {
-            "version": "0.2.0",
-            "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.2.0.tgz",
-            "integrity": "sha1-oOahcV2u0GLzIrYit/5a/RA1tuI="
-        },
-        "pkginfo": {
-            "version": "0.3.1",
-            "resolved": "https://registry.npmjs.org/pkginfo/-/pkginfo-0.3.1.tgz",
-            "integrity": "sha1-Wyn2qB9wcXFC4J52W76rl7T4HiE="
-        },
-        "qs": {
-            "version": "0.5.6",
-            "resolved": "https://registry.npmjs.org/qs/-/qs-0.5.6.tgz",
-            "integrity": "sha1-MbGtBYVnZRxSaSFQa5qHk5EaA4Q="
-        },
-        "readable-stream": {
-            "version": "1.0.31",
-            "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.31.tgz",
-            "integrity": "sha1-jyUC4LyeOw2huUUgqrtOJgPsr64="
-        },
-        "request": {
-            "version": "2.16.6",
-            "resolved": "https://registry.npmjs.org/request/-/request-2.16.6.tgz",
-            "integrity": "sha1-hy/kRa5y3iZrN4edatfclI+gHK0="
-        },
-        "require_optional": {
-            "version": "1.0.0",
-            "resolved": "https://registry.npmjs.org/require_optional/-/require_optional-1.0.0.tgz",
-            "integrity": "sha1-UqhhN6hJco62ClVTNhf4+RT1mr8="
-        },
-        "require-all": {
-            "version": "0.0.8",
-            "resolved": "https://registry.npmjs.org/require-all/-/require-all-0.0.8.tgz",
-            "integrity": "sha1-p9QwfZDkIvy58ErwGMFJkgB05LM="
-        },
-        "resolve-from": {
-            "version": "2.0.0",
-            "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-2.0.0.tgz",
-            "integrity": "sha1-lICrIOlP+h2egKgEx+oUdhGWa1c="
-        },
-        "semver": {
-            "version": "5.3.0",
-            "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz",
-            "integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8="
-        },
-        "sntp": {
-            "version": "0.1.4",
-            "resolved": "https://registry.npmjs.org/sntp/-/sntp-0.1.4.tgz",
-            "integrity": "sha1-XvSBuVGnspr/30r9fyaDj8ESD4Q="
-        },
-        "stack-trace": {
-            "version": "0.0.10",
-            "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz",
-            "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA="
-        },
-        "string_decoder": {
-            "version": "0.10.31",
-            "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
-            "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ="
-        },
-        "tunnel-agent": {
-            "version": "0.2.0",
-            "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.2.0.tgz",
-            "integrity": "sha1-aFPCr7GyEJ5FYp5JK9419Fnqaeg="
-        },
-        "tv4": {
-            "version": "1.3.0",
-            "resolved": "https://registry.npmjs.org/tv4/-/tv4-1.3.0.tgz",
-            "integrity": "sha1-0CDIRvrdUMhVq7JeuuzGj8EPeWM="
-        },
-        "winston": {
-            "version": "0.7.3",
-            "resolved": "https://registry.npmjs.org/winston/-/winston-0.7.3.tgz",
-            "integrity": "sha1-euMTunP83C7LSqL5zURugphncmY="
-        },
-        "winston-mysql-transport": {
-            "version": "0.1.2",
-            "resolved": "https://registry.npmjs.org/winston-mysql-transport/-/winston-mysql-transport-0.1.2.tgz",
-            "integrity": "sha1-ZHtpxVW2P5Y0HTER6kWWKxRHyIM="
-        }
-    }
-}
diff --git a/etl/js/package.json b/etl/js/package.json
index 99234ccac3..eae39f81c8 100644
--- a/etl/js/package.json
+++ b/etl/js/package.json
@@ -11,8 +11,7 @@
         "mongodb": "3.1.x",
         "mysql": "2.2.x",
         "tv4": "^1.0.13",
-        "winston": "0.7.x",
-        "winston-mysql-transport": "0.1.x"
+        "winston": "3.1.0"
     },
     "engine": "node >= 0.10.25"
 }