Merge pull request #1 from ubccr/xdmod7.0

Update from latest changes
ubccr · Jun 29, 2017 · c08959a · c08959a
2 parents d7c84c1 + ba7f68f
commit c08959a
Show file tree

Hide file tree

Showing 181 changed files with 13,250 additions and 6,079 deletions.
diff --git a/.travis.build.sh b/.travis.build.sh
@@ -112,7 +112,7 @@ elif [ "$TEST_SUITE" = "style" ]; then
         fi
     done
 elif [ "$TEST_SUITE" = "unit" ]; then
-    open_xdmod/modules/xdmod/tests/runtests.sh
+    open_xdmod/modules/xdmod/tests/runtests.sh && phantomjs html/unit_tests/phantom.js
     if [ $? != 0 ]; then
         build_exit_value=2
     fi

diff --git a/.travis.install.sh b/.travis.install.sh
@@ -36,4 +36,9 @@ pear install Log
 source ~/.nvm/nvm.sh
 nvm install "$NODE_VERSION"
 nvm use "$NODE_VERSION"
+
+echo "Updating npm..."
+npm update -g npm
+
+echo "Installing npm dependencies..."
 npm install
diff --git a/.travis.yml b/.travis.yml
@@ -9,18 +9,24 @@ php:
 env:
     global:
         - NODE_VERSION=6
+        - XDMOD_TEST_ARTIFACTS_MIRROR="$HOME/xdmod-test-artifacts.git"
     matrix:
         - TEST_SUITE=syntax
         - TEST_SUITE=style
         - TEST_SUITE=unit
         - TEST_SUITE=build
+matrix:
+    exclude:
+        - php: '5.4'
+          env: TEST_SUITE=style
 
 # Add dependency directories to the Travis cache
 cache:
     directories:
         - $HOME/.npm
         - $HOME/.composer/cache
         - /tmp/pear/cache
+        - $XDMOD_TEST_ARTIFACTS_MIRROR
 
 # Delegate the installation step to the custom Travis installation script
 install: ./.travis.install.sh

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,117 @@
 Open XDMoD Change Log
 =====================
 
+2017-05-11 v6.6.0
+-----------------
+
+- Features
+    - General
+        - Added ability to group realms together under categories
+          ([\#60](https://github.com/ubccr/xdmod/pull/60))
+          - Categories have taken the place of realms in the user interface
+        - Improved support for browser client assets provided by modules
+          ([\#82](https://github.com/ubccr/xdmod/pull/82),
+           [\#113](https://github.com/ubccr/xdmod/pull/113))
+        - Improved upgrade messaging
+          ([\#86](https://github.com/ubccr/xdmod/pull/86))
+    - ETLv2
+        - Added per-pipeline ETL locks
+          ([\#10](https://github.com/ubccr/xdmod/pull/10))
+        - Enhanced multi-host aggregation
+          ([\#13](https://github.com/ubccr/xdmod/pull/13))
+        - Added read support for Oracle endpoints
+          ([\#34](https://github.com/ubccr/xdmod/pull/34))
+        - Improved support for running pipelines over all dates
+          ([\#77](https://github.com/ubccr/xdmod/pull/77))
+        - Improved transformation support
+          ([\#80](https://github.com/ubccr/xdmod/pull/80))
+        - Added a tool for comparing SQL tables
+          ([\#78](https://github.com/ubccr/xdmod/pull/78))
+        - Added other improvements
+          ([\#43](https://github.com/ubccr/xdmod/pull/43),
+           [\#45](https://github.com/ubccr/xdmod/pull/45),
+           [\#84](https://github.com/ubccr/xdmod/pull/84),
+           [\#90](https://github.com/ubccr/xdmod/pull/90),
+           [\#92](https://github.com/ubccr/xdmod/pull/92))
+- Bug Fixes
+    - General
+        - Fixed warning that could appear when using federated authentication
+          ([\#19](https://github.com/ubccr/xdmod/pull/19))
+        - Fixed unnecessary rounding in processor count statistics
+          ([\#25](https://github.com/ubccr/xdmod/pull/25))
+        - Fixed errors when attempting to preview an unsaved report
+          ([\#26](https://github.com/ubccr/xdmod/pull/26))
+        - Fixed aggregators excluding data on certain time boundaries
+          ([\#47](https://github.com/ubccr/xdmod/pull/47))
+        - Fixed handling of backslashes when using certain MySQL features
+          ([\#52](https://github.com/ubccr/xdmod/pull/52),
+           [\#53](https://github.com/ubccr/xdmod/pull/53))
+        - Fixed non-aggregate metrics being allowed in aggregate datasets
+          ([\#74](https://github.com/ubccr/xdmod/pull/74))
+        - Fixed REST API throwing non-standard errors
+          ([\#87](https://github.com/ubccr/xdmod/pull/87))
+        - Fixed display error with password reset form that occurred when using
+          federated authentication
+          ([\#108](https://github.com/ubccr/xdmod/pull/108))
+        - Fixed "Show chart title" option in Usage tab not working as expected
+          ([\#139](https://github.com/ubccr/xdmod/pull/139))
+    - Metric Explorer
+        - Fixed Y-axis context menu disappearing after
+          changing between linear and log scales
+          ([\#12](https://github.com/ubccr/xdmod/pull/12))
+        - Fixed typing certain characters causing chart options menu to close
+          when using Firefox
+          ([\#119](https://github.com/ubccr/xdmod/pull/119))
+        - Fixed chart errors not displaying if help graphic is active
+          ([\#121](https://github.com/ubccr/xdmod/pull/121))
+        - Fixed chart last modified time not updating
+          ([\#140](https://github.com/ubccr/xdmod/pull/140))
+- Miscellaneous
+    - Performed work in anticipation of allocations/accounts data in Open XDMoD
+      ([\#11](https://github.com/ubccr/xdmod/pull/11),
+       [\#50](https://github.com/ubccr/xdmod/pull/50))
+    - Performed work in anticipation of cloud data in Open XDMoD
+      ([\#68](https://github.com/ubccr/xdmod/pull/68),
+       [\#75](https://github.com/ubccr/xdmod/pull/75))
+    - Cleaned up old and/or unused code
+      ([\#54](https://github.com/ubccr/xdmod/pull/54),
+       [\#55](https://github.com/ubccr/xdmod/pull/55),
+       [\#56](https://github.com/ubccr/xdmod/pull/56),
+       [\#57](https://github.com/ubccr/xdmod/pull/57),
+       [\#73](https://github.com/ubccr/xdmod/pull/73),
+       [\#88](https://github.com/ubccr/xdmod/pull/88))
+    - Improved logging
+      ([\#29](https://github.com/ubccr/xdmod/pull/29),
+       [\#40](https://github.com/ubccr/xdmod/pull/40),
+       [\#46](https://github.com/ubccr/xdmod/pull/46),
+       [\#50](https://github.com/ubccr/xdmod/pull/50),
+       [\#66](https://github.com/ubccr/xdmod/pull/66),
+       [\#98](https://github.com/ubccr/xdmod/pull/98))
+    - Improved quality assurance
+      ([\#21](https://github.com/ubccr/xdmod/pull/21),
+       [\#27](https://github.com/ubccr/xdmod/pull/27),
+       [\#28](https://github.com/ubccr/xdmod/pull/28),
+       [\#35](https://github.com/ubccr/xdmod/pull/35),
+       [\#41](https://github.com/ubccr/xdmod/pull/41),
+       [\#48](https://github.com/ubccr/xdmod/pull/48),
+       [\#58](https://github.com/ubccr/xdmod/pull/58),
+       [\#67](https://github.com/ubccr/xdmod/pull/67),
+       [\#76](https://github.com/ubccr/xdmod/pull/76),
+       [\#79](https://github.com/ubccr/xdmod/pull/79),
+       [\#93](https://github.com/ubccr/xdmod/pull/93),
+       [\#94](https://github.com/ubccr/xdmod/pull/94))
+    - Improved documentation
+      ([\#32](https://github.com/ubccr/xdmod/pull/32),
+       [\#37](https://github.com/ubccr/xdmod/pull/37),
+       [\#44](https://github.com/ubccr/xdmod/pull/44),
+       [\#71](https://github.com/ubccr/xdmod/pull/71),
+       [\#103](https://github.com/ubccr/xdmod/pull/103),
+       [\#115](https://github.com/ubccr/xdmod/pull/115),
+       [\#123](https://github.com/ubccr/xdmod/pull/123),
+       [\#130](https://github.com/ubccr/xdmod/pull/130),
+       [\#132](https://github.com/ubccr/xdmod/pull/132),
+       [\#135](https://github.com/ubccr/xdmod/pull/135))
+
 2017-01-10 v6.5.0
 -----------------
 

diff --git a/README.md b/README.md
@@ -199,6 +199,7 @@ may be found [here][repo-format]. Once you have made the desired changes, run
     - Ubuntu 16.04
       - `php-curl`
       - `php-gmp`
+      - `php-zip`
   - [Composer](https://getcomposer.org/)
   - [PEAR](https://pear.php.net/)
   - [PEAR Log Module](https://pear.php.net/package/Log/)

diff --git a/bin/xdmod-slurm-helper b/bin/xdmod-slurm-helper
@@ -242,7 +242,7 @@ function getSacctCmdArgs(
         $args[] = '--clusters';
         $args[] = $shredder->getResource();
     } else {
-        $args[] = '--allclusters';
+        throw new Exception('Shredder resource not set.');
     }
 
     $utc = new DateTimeZone('UTC');
@@ -321,14 +321,9 @@ Usage: xdmod-slurm-helper [-v] [-r resource]
         Output warning level logging.
 
     -r, --resource *resource*
-        Shred accounting data for only the specified cluster.  This
-        value will be passed to the sacct --clusters option.  If
-        this option is not specified all clusters will be shredded using
-        the sacct --allclusters option.  You should consistently
-        always or never use this option if you are not explicitly
-        specifying a time range since the cluster name is used to
-        determine the date range.  Unlike the sacct command, only one
-        cluster should be specified.
+        Shred accounting data for the specified cluster.  This value
+        will be passed to the sacct --clusters option.  Unlike the sacct
+        command, only one cluster should be specified.
 
     --start-time *datetime*
         Specify the start date and time to shred from sacct  The

diff --git a/bin/xdmod-upgrade b/bin/xdmod-upgrade
@@ -15,6 +15,7 @@ use CCR\DB\MySQLHelper;
 use CCR\Log;
 use Xdmod\Version;
 use OpenXdmod\Migration\MigrationFactory;
+use OpenXdmod\Setup\Console;
 
 // Disable memory limit.
 ini_set('memory_limit', -1);
@@ -25,7 +26,7 @@ ini_set('memory_limit', -1);
  * @var array
  */
 $supportedUpgrades = array(
-    '6.7.0' => '7.0.0',
+    '6.6.0' => '7.0.0',
 );
 
 /**
@@ -58,6 +59,8 @@ function main()
 {
     global $argv, $logger, $supportedUpgrades, $installedVersion;
 
+    $interactiveMode = true;
+
     $opts = array(
         array('h', 'help'),
         array('v', 'verbose'),
@@ -66,6 +69,7 @@ function main()
         array('',  'config-files'),
         array('',  'databases'),
         array('',  'current-version:'),
+        array('',  'batch-mode'),
     );
 
     $shortOptions = implode(
@@ -122,6 +126,9 @@ function main()
             case 'current-version':
                 $configFileVersion = $value;
                 break;
+            case 'batch-mode':
+                $interactiveMode = false;
+                break;
             default:
                 fwrite(STDERR, "Unexpected option '$key'\n");
                 exit(1);
@@ -183,10 +190,15 @@ function main()
         exit(1);
     }
 
-    displayWarning($configFileVersion, $installedVersion);
-    $response = readline('Are you sure you want to continue (yes/no): ');
-    if ($response != 'yes') {
-        exit;
+    $console = Console::factory();
+    $console->setInteractive($interactiveMode);
+
+    if ($interactiveMode) {
+        displayWarning($configFileVersion, $installedVersion);
+        $response = readline('Are you sure you want to continue (yes/no): ');
+        if ($response != 'yes') {
+            exit;
+        }
     }
 
     while (isset($supportedUpgrades[$configFileVersion])) {
@@ -295,6 +307,11 @@ Usage: xdmod-upgrade [-v]
         the version number in portal_settings.ini does not match the
         version of your config files and databases.
 
+    --batch-mode
+        Automatically run the upgrade process without any interactive
+        prompts. WARNING THIS WILL UPDATE THE FILES AND DATABASE WITHOUT
+        ANY OTHER CONFIRMATION PROMPTS.
+
 EOF;
 }
 
diff --git a/classes/Authentication/SAML/XDSamlAuthentication.php b/classes/Authentication/SAML/XDSamlAuthentication.php
@@ -2,6 +2,7 @@
 
 namespace Authentication\SAML;
 
+use CCR\MailWrapper;
 use \Exception;
 
 class XDSamlAuthentication
@@ -96,11 +97,16 @@ public function getXdmodAccount()
             $newUser->setUserType(FEDERATED_USER_TYPE);
             try {
                 $newUser->saveUser();
+            } catch(Exception $e) {
+                error_log('User creation failed: ' . $e->getMessage());
+                return false;
+            }
+            try {
                 self::notifyAdminOfNewUser($newUser, $samlAttrs, ($personId != -2));
                 return $newUser;
             } catch (Exception $e) {
-                self::notifyAdminOfNewUser($newUser, $samlAttrs, ($personId != -2), true);
-                return "EXCEPTION";
+                error_log("Error notifying " . $newUser->getEmailAddress() . ": " . $e->getMessage());
+                return false;
             }
         }
         return false;
@@ -136,40 +142,40 @@ public function getLoginLink()
     }
     private function notifyAdminOfNewUser($user, $samlAttributes, $linked, $error = false)
     {
-        $mail = ZendMailWrapper::init();
+        $userEmail = $user->getEmailAddress();
+        $userName = $user->getFormalName();
+
+        $mail = MailWrapper::initPHPMailer($userEmail, $userName);
 
         $recipient
         = (xd_utilities\getConfiguration('general', 'debug_mode') == 'on')
         ? xd_utilities\getConfiguration('general', 'debug_recipient')
         : xd_utilities\getConfiguration('general', 'contact_page_recipient');
-        $mail->addTo($recipient);
+        $mail->addAddress($recipient);
         if ($error) {
-            $mail->setSubject("[xdmod] Error Creating federated user");
+            $mail->Subject = "[xdmod] Error Creating federated user";
         } else {
-            $mail->setSubject("[xdmod] New ". ($linked ? "linked": "unlinked") ." federated user created");
+            $mail->Subject = "[xdmod] New " . ($linked ? "linked": "unlinked") . " federated user created";
         }
 
-        $userEmail = $user->getEmailAddress();
         if ($userEmail != NO_EMAIL_ADDRESS_SET) {
-            $mail->setFrom($userEmail);
-            $mail->setReplyTo($userEmail);
+            $mail->addReplyTo($userEmail, $userName);
         }
 
         $body = "The following person has had an account created on XDMoD:\n\n" .
         "Person Details ----------------------------------\n\n" .
         "\nName:                     " . $user->getFormalName(true) .
         "\nUserame:                  " . $user->getUsername() .
-        "\nE-Mail:                   " . $userEmail .
-        "\n\n" .
-        "Additional SAML Attributes ----------------------------------\n\n" .
-        print_r($samlAttributes, true);
+        "\nE-Mail:                   " . $userEmail ;
 
-        $mail->setBodyText($body);
-        try {
-            $mail->send();
-            return true;
-        } catch (Exception $e) {
-            return $e->getMessage();
+        if(count($samlAttributes) != 0) {
+            $body = $body . "\n\n" .
+                "Additional SAML Attributes ----------------------------------\n\n" .
+                print_r($samlAttributes, true);
         }
+
+        $mail->Body = $body;
+
+        $mail->send();
     }
 }
diff --git a/classes/CCR/Log.php b/classes/CCR/Log.php
@@ -73,7 +73,7 @@ public static function factory(
 
         $loggers = self::getLoggers($ident, $conf);
 
-        $logger = \Log::singleton('composite');
+        $logger = \Log::factory('composite');
 
         foreach ($loggers as $childLogger) {
             $logger->addChild($childLogger);

diff --git a/classes/CCR/MailWrapper.php b/classes/CCR/MailWrapper.php
@@ -0,0 +1,31 @@
+<?php
+
+namespace CCR;
+
+class MailWrapper
+{
+
+    public static function initPHPMailer($fromEmail = null, $fromName = null)
+    {
+        $mail = new \PHPMailer(true);
+        $mail->isSendMail();
+        $address =  \xd_utilities\getConfiguration('mailer', 'sender_email');
+        $mail->Sender = $address;
+
+        if($fromEmail !== null) {
+            $address = $fromEmail;
+            $name = $fromName;
+        } else {
+            $name = \xd_utilities\getConfiguration('general', 'title');
+        }
+
+        try {
+            $mail->setFrom($address, $name);
+        } catch(phpmailerException $e) {
+            error_log($e->getMessage());
+            throw $e;
+        }
+
+        return $mail;
+    }
+}