From 133111a63ea912e259133ad5eb8434f3dde756f6 Mon Sep 17 00:00:00 2001 From: Steve Gallo Date: Fri, 13 Jan 2017 13:50:53 -0500 Subject: [PATCH 01/15] Improve rdbms dataendpoint error messages --- classes/CCR/DB/SybaseDB.php | 58 --------------------- classes/ETL/DataEndpoint/aRdbmsEndpoint.php | 16 +++++- tools/etl/etl_overseer.php | 12 ++++- 3 files changed, 25 insertions(+), 61 deletions(-) delete mode 100644 classes/CCR/DB/SybaseDB.php diff --git a/classes/CCR/DB/SybaseDB.php b/classes/CCR/DB/SybaseDB.php deleted file mode 100644 index 24e5db451b..0000000000 --- a/classes/CCR/DB/SybaseDB.php +++ /dev/null @@ -1,58 +0,0 @@ -_dbh) - { - - try { - - $dsn = $this->_db_engine . ':host=' . $this->_db_host . ':' . $this->_db_port. ';dbname=' . $this->_db_name; - - $this->_dbh = new PDO($dsn, $this->_db_username, $this->_db_password); - $this->_dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); - - } - catch (PDOException $err) - { - throw $err; - } - - } - - return $this->_dbh; - - }// connect() - -}//SybaseDB - -?> diff --git a/classes/ETL/DataEndpoint/aRdbmsEndpoint.php b/classes/ETL/DataEndpoint/aRdbmsEndpoint.php index 03ff5e8f40..2ebcabdf3d 100644 --- a/classes/ETL/DataEndpoint/aRdbmsEndpoint.php +++ b/classes/ETL/DataEndpoint/aRdbmsEndpoint.php @@ -32,6 +32,9 @@ abstract class aRdbmsEndpoint extends aDataEndpoint // Database hostname protected $hostname = null; + // Database port + protected $port = null; + // User used to connect to the database protected $username = null; @@ -60,6 +63,9 @@ public function __construct(DataEndpointOptions $options, Log $logger = null) if ( array_key_exists("host", $section) ) { $this->hostname = $section["host"]; } + if ( array_key_exists("port", $section) ) { + $this->port = $section["port"]; + } if ( array_key_exists("user", $section) ) { $this->username = $section["user"]; } @@ -123,8 +129,15 @@ public function connect() // DB::factory() does not support connecting to an alternate schema and needs a separate entry // in the config file. You can, however, explicitly reference a schema in your query. - $this->handle = DB::factory($this->config); + try { + $this->handle = DB::factory($this->config); + } catch (Exception $e) { + $msg = "Error connecting to data endpoint '" . $this->name . "'. " . $e->getMessage(); + $this->logAndThrowException($msg); + } + return $this->handle; + } // connect() /* ------------------------------------------------------------------------------------------ @@ -262,6 +275,7 @@ public function __toString() { return "{$this->name} (" . get_class($this) . ", config={$this->config}, schema={$this->schema}" . (null !== $this->hostname ? ", host={$this->hostname}" : "" ) . + (null !== $this->port ? ":{$this->port}" : "" ) . (null !== $this->username ? ", user={$this->username}" : "" ) . ")"; } // __toString() diff --git a/tools/etl/etl_overseer.php b/tools/etl/etl_overseer.php index 9d556c11bf..ab57a91b75 100755 --- a/tools/etl/etl_overseer.php +++ b/tools/etl/etl_overseer.php @@ -392,7 +392,11 @@ case 'list-resources': $sql = "SELECT code, start_date, end_date from {$utilitySchema}.resourcefact WHERE resourcetype_id NOT IN (0,4) ORDER BY CODE ASC"; - $result = $utilityEndpoint->getHandle()->query($sql); + try { + $result = $utilityEndpoint->getHandle()->query($sql); + } catch (Exception $e) { + exit($e->getMessage() . "\n". $e->getTraceAsString() . "\n"); + } $headings = array("Resource Code","Start Date","End Date"); print implode(LIST_SEPARATOR, $headings) . "\n"; @@ -519,7 +523,11 @@ // Look up resource ids and generate the mapping for resource codes to ids. This can be stored in // the overseer and used by actions if needed. -$result = $utilityEndpoint->getHandle()->query("SELECT id, code from {$utilitySchema}.resourcefact"); +try { + $result = $utilityEndpoint->getHandle()->query("SELECT id, code from {$utilitySchema}.resourcefact"); +} catch (Exception $e) { + exit($e->getMessage() . "\n". $e->getTraceAsString() . "\n"); +} $scriptOptions['resource-code-map'] = array(); foreach ( $result as $row ) { From 2c82164d24ea52e25245493f19190338a933eace Mon Sep 17 00:00:00 2001 From: Steve Gallo Date: Fri, 13 Jan 2017 14:00:18 -0500 Subject: [PATCH 02/15] Prepare for adding Oracle support - More flexible database parameters in .ini file (not all databases require all parameters) - Improve handling of DB Engine subclasses (e.g., PostgresDB, MySQLDB) - Improved parameter validation on a per-engine basis - Added methods to iDatabase that should have been there before - General code cleanup - Improve comments --- classes/CCR/DB.php | 151 +++++++++++++++++++--------------- classes/CCR/DB/MySQLDB.php | 26 +++--- classes/CCR/DB/PDODB.php | 148 +++++++++++++++------------------ classes/CCR/DB/PostgresDB.php | 25 ++++-- classes/CCR/DB/iDatabase.php | 117 +++++++++++++++++++------- 5 files changed, 272 insertions(+), 195 deletions(-) diff --git a/classes/CCR/DB.php b/classes/CCR/DB.php index 0c4fe041f5..b5f171ca28 100644 --- a/classes/CCR/DB.php +++ b/classes/CCR/DB.php @@ -19,75 +19,94 @@ namespace CCR; use xd_utilities; -use \Exception; +use Exception; use CCR\DB\iDatabase; class DB { - // An array (pool) of database connection handles, one per configuration file section - - private static $instancePool = array(); - - // Ensure that this class is a singleton - - private function __construct() {} - - // ================================================================================ - // Cleanup - // ================================================================================ - - public function __destruct() {} - - // ================================================================================ - // Create an instance of the database singleton. A single argument is - // required, which is configuration file section identifier (e.g. [datawarehouse]). - // The database connection parameters in that section will be used to create the - // instance, which will be cached for re-use by subsequent requests targeting the - // same section. - // - // @throws Exception if there is an invalid number of arguments - // - // @returns An instance of the database class - // ================================================================================ - - public static function factory($section, $autoConnect = true) - { - // If this section has been used before in creating a database instance (handle), then - // it will have been cached. In this case, the cached handle will be returned. - - if ( array_key_exists($section, self::$instancePool) ) { - return self::$instancePool[$section]; - } - - $engine = xd_utilities\getConfiguration($section, 'db_engine'); - $host = xd_utilities\getConfiguration($section, 'host'); - $database = xd_utilities\getConfiguration($section, 'database'); - $user = xd_utilities\getConfiguration($section, 'user'); - $passwd = xd_utilities\getConfiguration($section, 'pass'); - $port = xd_utilities\getConfiguration($section, 'port'); - - $engine = "CCR\\DB\\$engine"; - - // Ensure that the class exists before we attempt to instantiate it - - if ( class_exists($engine) ) { - $db = new $engine($host, $port, $database, $user, $passwd); - } else { - $msg = "Error creating database '" . $options->name . "', class '$className' not found"; - throw new Exception($msg); - } - - // All database interfaces must implement iDatabase - - if ( ! $db instanceof iDatabase ) { - throw new Exception("$engine does not implenment interface iDatabase"); - } - - self::$instancePool[$section] = $db; - if ($autoConnect) self::$instancePool[$section]->connect(); - - return self::$instancePool[$section]; - - } // factory() + // An array (pool) of database connection handles, one per configuration file section + + private static $instancePool = array(); + + // Ensure that this class is a singleton + + private function __construct() {} + + // ================================================================================ + // Cleanup + // ================================================================================ + + public function __destruct() {} + + // ================================================================================ + // Create an instance of the database singleton. A single argument is + // required, which is configuration file section identifier (e.g. [datawarehouse]). + // The database connection parameters in that section will be used to create the + // instance, which will be cached for re-use by subsequent requests targeting the + // same section. + // + // @param $sectionName Name of the configuration section containing database parameters + // @param $autoConnect If TRUE, connect to the database after creating the object + // + // @throws Exception if there is an invalid number of arguments + // + // @returns An instance of the database class + // ================================================================================ + + public static function factory($sectionName, $autoConnect = true) + { + // If this section has been used before in creating a database instance (handle), then + // it will have been cached. In this case, the cached handle will be returned. + + if ( array_key_exists($sectionName, self::$instancePool) ) { + return self::$instancePool[$sectionName]; + } + + try { + $iniSection = xd_utilities\getConfigurationSection($sectionName, 'db_engine'); + } catch (Exception $e) { + $msg = "Unable to get database configuration options: " . $e->getMessage(); + throw new Exception ($msg); + } + + // Not all engines are required to specify all configuration options (e.g., Oracle) so + // allow NULL (empty) options. Specific engines may enforce additional requirements. + + $engine = ( array_key_exists('db_engine', $iniSection) ? $iniSection['db_engine'] : null); + $database = ( array_key_exists('database', $iniSection) ? $iniSection['database'] : null); + $user = ( array_key_exists('user', $iniSection) ? $iniSection['user'] : null ); + + if ( null === $engine || null === $database || null === $user ) { + $msg = "Configuration section '$sectionName' missing required options (db_engine, database, user)"; + throw new Exception($msg); + } + + $password = ( array_key_exists('pass', $iniSection) ? $iniSection['pass'] : null ); + $host = ( array_key_exists('host', $iniSection) ? $iniSection['host'] : null ); + $port = ( array_key_exists('port', $iniSection) ? $iniSection['port'] : null ); + + $engine = "CCR\\DB\\$engine"; + + // Ensure that the class exists before we attempt to instantiate it + + if ( class_exists($engine) ) { + $db = new $engine($host, $port, $database, $user, $password); + } else { + $msg = "Error creating database defined in section '$sectionName', class '$engine' not found"; + throw new Exception($msg); + } + + // All database interfaces must implement iDatabase + + if ( ! $db instanceof iDatabase ) { + throw new Exception("$engine does not implenment interface iDatabase"); + } + + self::$instancePool[$sectionName] = $db; + if ($autoConnect) self::$instancePool[$sectionName]->connect(); + + return self::$instancePool[$sectionName]; + + } // factory() } // class DB diff --git a/classes/CCR/DB/MySQLDB.php b/classes/CCR/DB/MySQLDB.php index bc8ca2b90d..9d90671082 100644 --- a/classes/CCR/DB/MySQLDB.php +++ b/classes/CCR/DB/MySQLDB.php @@ -1,10 +1,10 @@ @@ -13,18 +13,22 @@ namespace CCR\DB; +use Exception; + class MySQLDB extends PDODB implements iDatabase { - function __construct($db_host,$db_port,$db_name,$db_username,$db_password) + // ------------------------------------------------------------------------------------------ + // @see iDatabase::__construct() + // ------------------------------------------------------------------------------------------ + + function __construct($db_host, $db_port, $db_name, $db_username, $db_password, $dsn_extra = null) { - $dsn = 'mysql:host=' . $db_host . ';port=' . $db_port . ';dbname=' . $db_name . ';charset=utf8'; - parent::__construct("mysql",$db_host,$db_port,$db_name,$db_username,$db_password, $dsn); + if ( null == $db_host || null === $db_name || null === $db_username ) { + $msg = "Database engine " . __CLASS__ . " requires (host, database, username)"; + throw new Exception($msg); + } + parent::__construct("mysql", $db_host, $db_port, $db_name, $db_username, $db_password, 'charset=utf8'); } - function __destruct() - { - parent::__destruct(); - } - -} +} // class MySQLDB diff --git a/classes/CCR/DB/PDODB.php b/classes/CCR/DB/PDODB.php index d6a43b00ef..92575ba151 100644 --- a/classes/CCR/DB/PDODB.php +++ b/classes/CCR/DB/PDODB.php @@ -1,30 +1,35 @@ * - Added prepare() * - Moved empty query check from individual calls to prepare() since they all call it anyway + * + * 2017-01-12 Steve Gallo + * - Added generateDsn() + * - Renamed destroy() to disconnect() + * - Cleaned up connect() + * - Renamed $dsn_override to $dsn_extra to allow extra parameters to be added + * ========================================================================================== */ namespace CCR\DB; -use \PDO; -use \Exception; - -/* - * @class PDODB - * The implementation of the interface iDatabase based on PDO - */ +use PDO; +use Exception; class PDODB implements iDatabase { + + // Database connection parameters public $_db_engine = NULL; public $_db_host = NULL; public $_db_port = NULL; @@ -32,8 +37,10 @@ class PDODB public $_db_username = NULL; public $_db_password = NULL; - public $_dsn_override = NULL; + // Optional extra parameters to be added to the DSN + public $_dsn_extra = NULL; + // Handle to the PDO instance protected $_dbh = NULL; protected static $_debug_mode = false; @@ -41,17 +48,10 @@ class PDODB protected static $_params = array(); // -------------------------------------------------------------------------------- - // Constructor - // - // @param $db_engine PDO database engine name - // @param $db_host Database hostname - // @param $db_port Database port - // @param $db_name Database name - // @param $db_username Database username - // @param $db_password Database user password + // @see iDatabase::__construct() // -------------------------------------------------------------------------------- - public function __construct($db_engine, $db_host, $db_port, $db_name, $db_username, $db_password, $dsn_override = NULL) + public function __construct($db_engine, $db_host, $db_port, $db_name, $db_username, $db_password, $dsn_extra = null) { $this->_db_engine = $db_engine; $this->_db_host = $db_host; @@ -59,57 +59,47 @@ public function __construct($db_engine, $db_host, $db_port, $db_name, $db_userna $this->_db_name = $db_name; $this->_db_username = $db_username; $this->_db_password = $db_password; - $this->_dsn_override = $dsn_override; + $this->_dsn_extra = $dsn_extra; } // __construct() // -------------------------------------------------------------------------------- - - function __destruct() - { - $this->destroy(); - } - + // Perform any necessary cleanup when the object is destroyed // -------------------------------------------------------------------------------- - public function disconnect() + public function __destruct() { - if (NULL !== $this->_dbh) - $this->_dbh->close(); + $this->disconnect(); } // -------------------------------------------------------------------------------- - // Connect to the database using PDO - // - // @throws PDOException if an error ocurred connecting to the database - // - // @returns An instance of the PDO database handle + // See iDatabase::connect() // -------------------------------------------------------------------------------- public function connect() { - if (NULL === $this->_dbh) { - try { - $dsn = (NULL !== $this->_dsn_override) ? $this->_dsn_override : $this->_db_engine . ':host=' . $this->_db_host . ';port=' . $this->_db_port . ';dbname=' . $this->_db_name; + if ( null !== $this->_dbh) { + return $this->_dbh; + } - $this->_dbh = new PDO($dsn, $this->_db_username, $this->_db_password); - $this->_dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); + $dsn = $this->generateDsn(); + $this->_dbh = new PDO($dsn, $this->_db_username, $this->_db_password); + $this->_dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); - } catch (Exception $err) { - throw $err; - } - } return $this->_dbh; } // connect() + // -------------------------------------------------------------------------------- + // See iDatabase::disconnect() // -------------------------------------------------------------------------------- - public function destroy() + public function disconnect() { $this->_dbh = NULL; } + // -------------------------------------------------------------------------------- - // @returns The database handle + // @see iDatabase::handle() // -------------------------------------------------------------------------------- public function handle() @@ -117,8 +107,28 @@ public function handle() return $this->connect(); } + // ------------------------------------------------------------------------------------------ + // Generate a standard PDO DSN. This method may be overriden by a child class if needed. + // + // @return A PDO connection DSN + // ------------------------------------------------------------------------------------------ + + protected function generateDsn() + { + $dsn = $this->_db_engine + . ':host=' . $this->_db_host + . ( null !== $this->_db_port ? ';port=' . $this->_db_port : '' ) + . ';dbname=' . $this->_db_name; + + if ( null !== $this->_dsn_extra ) { + $dsn .= ( 0 !== strpos($this->_dsn_extra, ';') ? ';' : '' ) . $this->_dsn_extra; + } + + return $dsn; + } // generateDsn() + // -------------------------------------------------------------------------------- - // @see Database::query() + // @see iDatabase::query() // -------------------------------------------------------------------------------- public function query($query, array $params = array(), $returnStatement = false) @@ -130,7 +140,7 @@ public function query($query, array $params = array(), $returnStatement = false) } catch (Exception $e) { // TODO: setup the logger and log this. } - + if (FALSE === $stmt->execute($params)) { list($sqlState, $errorCode, $errorMsg) = $stmt->errorInfo; throw new Exception("$sqlState: $errorMsg ($errorCode)"); @@ -143,13 +153,13 @@ public function query($query, array $params = array(), $returnStatement = false) } // query() // -------------------------------------------------------------------------------- - // @see Database::execute() + // @see iDatabase::execute() // -------------------------------------------------------------------------------- public function execute($query, array $params = array()) { $stmt = $this->prepare($query); - + try { if ( $this->debugging() ) $this->debug($query, $params); } catch (Exception $e) { @@ -166,49 +176,33 @@ public function execute($query, array $params = array()) } // execute() // -------------------------------------------------------------------------------- - // Prepare a query for execution and return the prepared statement. - // - // @param $query The query string - // - // @throws Exception if the query string is empty - // @throws Exception if there was an error preparing the query - // - // @return The prepared PDOStatement + // @see iDatabase::prepare() // -------------------------------------------------------------------------------- - + public function prepare($query) { if (empty($query)) { throw new Exception("No query string provided"); } - + return $this->handle()->prepare($query); } // prepare() // -------------------------------------------------------------------------------- - // Perform an INSERT command - // - // @param $statement The insert statement / command - // @param $params An array of values with as many elements as there are bound - // parameters in the SQL statement being executed. - // - // @throws Exception if the statement string is empty - // @throws PDOException if table does not exist or other db errors - // - // @returns An integer referring to the id associated with the recently inserted record + // @see iDatabase::insert() // -------------------------------------------------------------------------------- public function insert($statement, $params = array()) { $stmt = $this->prepare($statement); - + try { if ( $this->debugging() ) $this->debug($statement, $params); } catch (Exception $e) { // TODO: setup the logger and log this. } - + if (FALSE === $stmt->execute($params)) { list($sqlState, $errorCode, $errorMsg) = $stmt->errorInfo; throw new Exception("$sqlState: $errorMsg ($errorCode)"); @@ -219,21 +213,13 @@ public function insert($statement, $params = array()) } // insert() // -------------------------------------------------------------------------------- - // Get the number of rows in a table - // - // @param $schema the schema the table belongs to - // @param $table the name of the table to count rows for - // - // @throws Exception if the table parameter is empty - // @throws PDOException if table does not exist or other db errors - - // @returns the number of rows in the table + // @see iDatabase::getRowCount() // -------------------------------------------------------------------------------- public function getRowCount($schema, $table) { if (empty($table)) { - throw new Exception("PDODB::getRowCount:: No table string provided"); + throw new Exception(__CLASS__ . ": No table string provided"); } $full_tablename = (empty($schema) ? '' : $schema . '.') . $table; diff --git a/classes/CCR/DB/PostgresDB.php b/classes/CCR/DB/PostgresDB.php index 14a4506e06..bdd100491e 100644 --- a/classes/CCR/DB/PostgresDB.php +++ b/classes/CCR/DB/PostgresDB.php @@ -1,5 +1,5 @@ * - Now implements iDatabase -* +* */ namespace CCR\DB; +use Exception; + class PostgresDB extends PDODB implements iDatabase { - function __construct($db_host,$db_port,$db_name,$db_username,$db_password) + // ------------------------------------------------------------------------------------------ + // @see iDatabase::__construct() + // ------------------------------------------------------------------------------------------ + + function __construct($db_host, $db_port, $db_name, $db_username, $db_password, $dsn_extra = null) { - parent::__construct("pgsql",$db_host,$db_port,$db_name,$db_username,$db_password); + if ( null == $db_host || null === $db_name || null === $db_username ) { + $msg = "Database engine " . __CLASS__ . " requires (host, database, username)"; + throw new Exception($msg); + } + + parent::__construct("pgsql", $db_host, $db_port, $db_name, $db_username, $db_password); } - function __destruct() - { - parent::__destruct(); - } -} +} // class PostgresDB diff --git a/classes/CCR/DB/iDatabase.php b/classes/CCR/DB/iDatabase.php index e62380ae67..0f99f6b71a 100644 --- a/classes/CCR/DB/iDatabase.php +++ b/classes/CCR/DB/iDatabase.php @@ -1,6 +1,6 @@ * - Added prepare() * - Changed name from Database to iDatabase for consistency with coding conventions + * + * 2017-01-12 Steve Gallo + * - Added methods that were added over time to PDODB (__construct(), handle(), insert(), + * getRowCount()) + * - Added documentation and cleaned up for consistency + * ========================================================================================== */ namespace CCR\DB; -/** - * The interface for database classes. - */ interface iDatabase { + /* ------------------------------------------------------------------------------------------ + * Constructor + * + * @param $db_engine PDO database engine name + * @param $db_host Database hostname + * @param $db_port Database port + * @param $db_name Database name + * @param $db_username Database username + * @param $db_password Database user password + * @param $dsn_extra Optional extra parameters to be added to the DSN + * ------------------------------------------------------------------------------------------ + */ - /** - * Establishes the connection to the database. - */ - public function connect(); + public function __construct($db_engine, $db_host, $db_port, $db_name, $db_username, $db_password); - /* - * Releases the connection to the database + /* ------------------------------------------------------------------------------------------ + * Perform any necessary cleanup when the object is destroyed, such as closing the + * connection + * ------------------------------------------------------------------------------------------ */ - public function destroy(); - /** - * @param string $statement (The SQL INSERT statement) - * @param array $params (The optional parameters to the database, - * when needed) + public function __destruct(); + + /* ------------------------------------------------------------------------------------------ + * Establishes the connection to the database. * - * @return int (Returns the index of the recently inserted record) + * @return A PDO database object + * ------------------------------------------------------------------------------------------ */ - public function insert($statement, $params = array()); - /** - * Perform a query and return an associative array of results. This - * is the recommended method for executing SELECT statements. + public function connect(); + + /* ------------------------------------------------------------------------------------------ + * Releases the connection to the database. + * ------------------------------------------------------------------------------------------ + */ + + public function disconnect(); + + /* ------------------------------------------------------------------------------------------ + * @return The the PDO handle created by connect() + * ------------------------------------------------------------------------------------------ + */ + + public function handle(); + + /* ------------------------------------------------------------------------------------------ + * Perform a query and return an associative array of results. This is the recommended + * method for executing SELECT statements. * * @param $query The query string * @param $params An array of values with as many elements as there @@ -51,14 +80,12 @@ public function insert($statement, $params = array()); * @throws Exception if there was an error executing the query * * @return An array containing the query results + * ------------------------------------------------------------------------------------------ */ - public function query( - $query, - array $params = array(), - $returnStatement = false - ); - /** + public function query($query, array $params = array(), $returnStatement = false); + + /* ------------------------------------------------------------------------------------------ * Execute an SQL statement and return the number of rows affected. * This is the recommended method for executing non-SELECT * statements. @@ -71,10 +98,12 @@ public function query( * @throws Exception if there was an error executing the query * * @returns The number of rows affected by the statement + * ------------------------------------------------------------------------------------------ */ + public function execute($query, array $params = array()); - /** + /* ------------------------------------------------------------------------------------------ * Prepare a query for execution and return the prepared statement. * * @param $query The query string @@ -83,10 +112,42 @@ public function execute($query, array $params = array()); * @throws Exception if there was an error preparing the query * * @return The prepared statement + * ------------------------------------------------------------------------------------------ */ public function prepare($query); + /* ------------------------------------------------------------------------------------------ + * Specifically support a query that is an INSERT command and return the last insert id. + * + * @param $statement The insert statement / command + * @param $params An array of values with as many elements as there are bound + * parameters in the SQL statement being executed. + * + * @throws Exception if the statement string is empty + * @throws PDOException if table does not exist or other db errors + * + * @returns An integer referring to the id associated with the recently inserted record + * ------------------------------------------------------------------------------------------ + */ + + public function insert($statement, $params = array()); + + /* ------------------------------------------------------------------------------------------ + * Return the number of rows in a table. + * + * @param $schema the schema the table belongs to + * @param $table the name of the table to count rows for + * + * @throws Exception if the table parameter is empty + * @throws PDOException if table does not exist or other db errors + * + * @returns the number of rows in the table + * ------------------------------------------------------------------------------------------ + */ + + public function getRowCount($schema, $table); + /** * Start a database transaction. * From e44857cd2d086d62d0b6e8b546768907dad136a0 Mon Sep 17 00:00:00 2001 From: Steve Gallo Date: Tue, 17 Jan 2017 15:42:57 -0500 Subject: [PATCH 03/15] Allow methods to override default schema name when checking if schema or table exists --- classes/ETL/DataEndpoint/Mysql.php | 28 +++++++++++++-------- classes/ETL/DataEndpoint/Postgres.php | 28 +++++++++++++-------- classes/ETL/DataEndpoint/aRdbmsEndpoint.php | 11 +++++--- classes/ETL/DataEndpoint/iRdbmsEndpoint.php | 18 ++++++------- 4 files changed, 50 insertions(+), 35 deletions(-) diff --git a/classes/ETL/DataEndpoint/Mysql.php b/classes/ETL/DataEndpoint/Mysql.php index d06f073f1f..1b7f366be9 100644 --- a/classes/ETL/DataEndpoint/Mysql.php +++ b/classes/ETL/DataEndpoint/Mysql.php @@ -12,8 +12,7 @@ use ETL\DataEndpoint\DataEndpointOptions; use \Log; -class Mysql extends aRdbmsEndpoint -implements iRdbmsEndpoint +class Mysql extends aRdbmsEndpoint implements iRdbmsEndpoint { /* ------------------------------------------------------------------------------------------ @@ -76,9 +75,11 @@ public function isSameServer(iDataEndpoint $cmp) * ------------------------------------------------------------------------------------------ */ - public function schemaExists($schemaName) + public function schemaExists($schemaName = null) { - if ( empty($schemaName) ) { + if ( null === $schemaName ) { + $schemaName = $this->getSchema(); + } elseif ( empty($schemaName) ) { $msg = "Schema name cannot be empty"; $this->logAndThrowException($msg); } @@ -88,7 +89,7 @@ public function schemaExists($schemaName) FROM information_schema.schemata WHERE schema_name = :schema"; - $params = array(":schema" => $this->getSchema()); + $params = array(":schema" => $schemaName); try { $dbh = $this->getHandle(); @@ -97,7 +98,7 @@ public function schemaExists($schemaName) return false; } } catch (\PdoException $e) { - $msg = "Error querying for schema '" . $this->getSchema() . "'"; + $msg = "Error querying for schema '$schemaName'"; $this->logAndThrowSqlException($sql, $e, $msg); } @@ -110,15 +111,21 @@ public function schemaExists($schemaName) * ------------------------------------------------------------------------------------------ */ - public function createSchema($schemaName) + public function createSchema($schemaName = null) { - if ( empty($schemaName) ) { + if ( null === $schemaName ) { + $schemaName = $this->getSchema(true); + } elseif ( empty($schemaName) ) { $msg = "Schema name cannot be empty"; $this->logAndThrowException($msg); + } else { + $schemaName = ( 0 !== strpos($schemaName, $this->systemQuoteChar) + ? $this->quoteSystemIdentifier($schemaName) + : $schemaName ); } // Don't use bind parameters because we don't want to quote the schema - $sql = "CREATE DATABASE IF NOT EXISTS " . $this->getSchema(true); + $sql = "CREATE DATABASE IF NOT EXISTS $schemaName"; // $params = array(":schema" => $this->getSchema()); @@ -126,12 +133,11 @@ public function createSchema($schemaName) $dbh = $this->getHandle(); $result = $dbh->execute($sql); } catch (\PdoException $e) { - $msg = "Error creating schema '" . $this->getSchema() . "'"; + $msg = "Error creating schema '$schemaName'"; $this->logAndThrowSqlException($sql, $e, $msg); } return true; } // createSchema() - } // class Mysql diff --git a/classes/ETL/DataEndpoint/Postgres.php b/classes/ETL/DataEndpoint/Postgres.php index f3a3ee19c3..fc5172f25e 100644 --- a/classes/ETL/DataEndpoint/Postgres.php +++ b/classes/ETL/DataEndpoint/Postgres.php @@ -12,8 +12,7 @@ use ETL\DataEndpoint\DataEndpointOptions; use \Log; -class Postgres extends aRdbmsEndpoint -implements iRdbmsEndpoint +class Postgres extends aRdbmsEndpoint implements iRdbmsEndpoint { public function __construct(DataEndpointOptions $options, Log $logger = null) @@ -40,9 +39,11 @@ public function isSameServer(iDataEndpoint $cmp) * ------------------------------------------------------------------------------------------ */ - public function schemaExists($schemaName) + public function schemaExists($schemaName = null) { - if ( empty($schemaName) ) { + if ( null === $schemaName ) { + $schemaName = $this->getSchema(); + } elseif ( empty($schemaName) ) { $msg = "Schema name cannot be empty"; $this->logAndThrowException($msg); } @@ -54,7 +55,7 @@ public function schemaExists($schemaName) FROM pg_catalog.pg_namespace WHERE nspname = :schema"; - $params = array(":schema" => $this->getSchema()); + $params = array(":schema" => $schemaName); try { $dbh = $this->getHandle(); @@ -63,7 +64,7 @@ public function schemaExists($schemaName) return false; } } catch (\PdoException $e) { - $msg = "Error querying for schema '" . $this->getSchema() . "'"; + $msg = "Error querying for schema '$schemaName'"; $this->logAndThrowSqlException($sql, $e, $msg); } @@ -76,15 +77,21 @@ public function schemaExists($schemaName) * ------------------------------------------------------------------------------------------ */ - public function createSchema($schemaName) + public function createSchema($schemaName = null) { - if ( empty($schemaName) ) { + if ( null === $schemaName ) { + $schemaName = $this->getSchema(true); + } elseif ( empty($schemaName) ) { $msg = "Schema name cannot be empty"; $this->logAndThrowException($msg); + } else { + $schemaName = ( 0 !== strpos($schemaName, $this->systemQuoteChar) + ? $this->quoteSystemIdentifier($schemaName) + : $schemaName ); } // Don't use bind parameters because we don't want to quote the schema - $sql = "CREATE SCHEMA IF NOT EXISTS " . $this->getSchema(true); + $sql = "CREATE SCHEMA IF NOT EXISTS $schemaName"; $params = array(":schema" => $this->getSchema()); @@ -92,12 +99,11 @@ public function createSchema($schemaName) $dbh = $this->getHandle(); $result = $dbh->query($sql, $params); } catch (\PdoException $e) { - $msg = "Error creating schema '" . $this->getSchema() . "'"; + $msg = "Error creating schema '$schemaName'"; $this->logAndThrowSqlException($sql, $e, $msg); } return true; } // createSchema() - } // class Postgres diff --git a/classes/ETL/DataEndpoint/aRdbmsEndpoint.php b/classes/ETL/DataEndpoint/aRdbmsEndpoint.php index 2ebcabdf3d..28723dc06a 100644 --- a/classes/ETL/DataEndpoint/aRdbmsEndpoint.php +++ b/classes/ETL/DataEndpoint/aRdbmsEndpoint.php @@ -228,7 +228,7 @@ public function tableExists($tableName, $schemaName = null) * ------------------------------------------------------------------------------------------ */ - public function getTableColumnNames($tableName) + public function getTableColumnNames($tableName, $schemaName = null) { if ( empty($tableName) ) { $msg = "Table name cannot be empty"; @@ -242,7 +242,11 @@ public function getTableColumnNames($tableName) AND table_name = :tablename ORDER BY ordinal_position ASC"; - $params = array(":schema" => $this->getSchema(), + if ( null === $schemaName ) { + $schemaName = $this->getSchema(); + } + + $params = array(":schema" => $schemaName, ":tablename" => $tableName); try { @@ -253,7 +257,7 @@ public function getTableColumnNames($tableName) $this->logAndThrowException($msg); } } catch (PDOException $e) { - $msg = "Error retrieving column names from '" . $this->getSchema() . ".'$tableName' "; + $msg = "Error retrieving column names from '$schemaName'$tableName' "; $this->logAndThrowSqlException($sql, $e, $msg); } @@ -279,5 +283,4 @@ public function __toString() (null !== $this->username ? ", user={$this->username}" : "" ) . ")"; } // __toString() - } // class aRdbmsEndpoint diff --git a/classes/ETL/DataEndpoint/iRdbmsEndpoint.php b/classes/ETL/DataEndpoint/iRdbmsEndpoint.php index b4047968fe..c0fbd8b194 100644 --- a/classes/ETL/DataEndpoint/iRdbmsEndpoint.php +++ b/classes/ETL/DataEndpoint/iRdbmsEndpoint.php @@ -15,7 +15,7 @@ interface iRdbmsEndpoint extends iDataEndpoint { - + /* ------------------------------------------------------------------------------------------ * Wrap a system identifier in quotes appropriate for the endpint. For example, MySQL uses a * backtick (`) to quote identifiers while Oracle and Postgres using double quotes ("). @@ -25,14 +25,14 @@ interface iRdbmsEndpoint extends iDataEndpoint * @return The identifier quoted appropriately for the endpoint * ------------------------------------------------------------------------------------------ */ - + public function quoteSystemIdentifier($identifier); /* ------------------------------------------------------------------------------------------ * @return The character for quoting system identifiers using this endpoint. * ------------------------------------------------------------------------------------------ */ - + public function getSystemQuoteChar(); /* ------------------------------------------------------------------------------------------ @@ -52,6 +52,7 @@ public function tableExists($tableName, $schemaName = null); * associated with the named table. * * @param $tableName The table to inspect + * @param $schemaName Optional schema name used to override the current DataEmdpoint schema * * @return An array of column names for the table * @@ -59,23 +60,23 @@ public function tableExists($tableName, $schemaName = null); * ------------------------------------------------------------------------------------------ */ - public function getTableColumnNames($tableName); + public function getTableColumnNames($tableName, $schemaName = null); /* ------------------------------------------------------------------------------------------ * Query the RDBMS and return TRUE if the schema exists, FALSE otherwise. * - * @param $schemaName The schema to inspect + * @param $schemaName The schema to inspector NULL current DataEndpoint schema * * @return TRUE if the schema exists. * ------------------------------------------------------------------------------------------ */ - public function schemaExists($schemaName); + public function schemaExists($schemaName = null); /* ------------------------------------------------------------------------------------------ * Create a schema. * - * @param $schemaName The schema to create + * @param $schemaName The schema to create, or NULL to use the current DataEndpoint schema * * @return TRUE if the schema was created. * @@ -83,7 +84,7 @@ public function schemaExists($schemaName); * ------------------------------------------------------------------------------------------ */ - public function createSchema($schemaName); + public function createSchema($schemaName = null); /* ------------------------------------------------------------------------------------------ * @param $quote TRUE to wrap the name in quotes to handle special characters @@ -94,5 +95,4 @@ public function createSchema($schemaName); */ public function getSchema($quote = false); - } // interface iRdbmsEndpoint From ba6862e513a88d372e0faa22d202407b162b8e52 Mon Sep 17 00:00:00 2001 From: Steve Gallo Date: Fri, 20 Jan 2017 13:18:48 -0500 Subject: [PATCH 04/15] Add support for ORDER BY in ETL queries --- classes/ETL/DbEntity/Query.php | 68 +++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/classes/ETL/DbEntity/Query.php b/classes/ETL/DbEntity/Query.php index 3d13423cf5..7c5e5be3df 100644 --- a/classes/ETL/DbEntity/Query.php +++ b/classes/ETL/DbEntity/Query.php @@ -63,6 +63,9 @@ class Query extends aNamedEntity // Optional array of WHERE clauses protected $where = array(); + // Optional array of ORDER BY fields + protected $orderBys = array(); + // Optional defined macros protected $macros = array(); @@ -178,6 +181,14 @@ protected function initialize(stdClass $config, $force = false) } } + if ( isset($config->orderby) ) { + if ( ! is_array($config->orderby) ) { + $errorMsg[] = "orderby property must be an array"; + } else if ( 0 == count($config->orderby) ) { + $errorMsg[] = "orderby property must include as least one element"; + } + } + if ( isset($config->where) && ! is_array($config->where) ) { $errorMsg[] = "where property must be an array"; } @@ -227,6 +238,12 @@ protected function initialize(stdClass $config, $force = false) } } + if ( isset($config->orderby) ) { + foreach ( $config->orderby as $orderby ) { + $this->addOrderBy($orderby); + } + } + if ( isset($config->macros) ) { foreach ( $config->macros as $macro ) { $this->addMacro($macro); @@ -379,10 +396,55 @@ public function getGroupBys() public function deleteGroupBys() { - $this->groupbys = array(); + $this->groupBys = array(); return $this; } // deleteGroupBys() + /* ------------------------------------------------------------------------------------------ + * Add a order by clause to this query. + * + * @param $orderBy An array containing the group by column names. + * + * @return This object to support method chaining. + * ------------------------------------------------------------------------------------------ + */ + + public function addOrderBy($orderBy) + { + if ( empty($orderBy) || ! is_string($orderBy) ) { + $msg = "Cannot add an empty group by"; + $this->logAndThrowException($msg); + } + + $this->orderBys[] = $orderBy; + return $this; + } // addOrderBys() + + /* ------------------------------------------------------------------------------------------ + * Get the list of order by columns. + * + * @return An array of group by column names. + * ------------------------------------------------------------------------------------------ + */ + + public function getOrderBys() + { + return $this->orderBys; + } // getOrderBys() + + /* ------------------------------------------------------------------------------------------ + * Remove all order bys from this query. + * + * @return This object to support method chaining. + * ------------------------------------------------------------------------------------------ + */ + + public function deleteOrderBys() + { + $this->orderBys = array(); + return $this; + } // deleteOrderBys() + /* ------------------------------------------------------------------------------------------ * Add a join clause for this query. * @@ -717,6 +779,7 @@ public function getSelectSql($includeSchema = true) implode(",\n", $columnList) . "\n" . implode("\n", $joinList) . "\n" . ( count($whereConditions) > 0 ? "WHERE " . implode("\nAND ", $whereConditions) . "\n" : "" ) . + ( count($this->orderBys) > 0 ? "ORDER BY " . implode(", ", $this->orderBys) : "" ); ( count($this->groupBys) > 0 ? "GROUP BY " . implode(", ", $this->groupBys) : "" ); // If any macros have been defined, process those macros now. Since macros can contain variables @@ -751,6 +814,9 @@ public function toJsonObj($succinct = false, $includeSchema = false) if ( count($this->groupBys) > 0 ) { $data->groupbys = $this->groupBys; } + if ( count($this->orderBys) > 0 ) { + $data->orderbys = $this->orderBys; + } if ( count($this->where) > 0 ) { $data->where = $this->where; } From 6d77b52c8d5b0983c10354f89c24e92d52937d07 Mon Sep 17 00:00:00 2001 From: Steve Gallo Date: Fri, 20 Jan 2017 13:20:19 -0500 Subject: [PATCH 05/15] Improve debug readability --- classes/ETL/Ingestor/pdoIngestor.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/ETL/Ingestor/pdoIngestor.php b/classes/ETL/Ingestor/pdoIngestor.php index 6f67faf13a..b5cb432e9b 100644 --- a/classes/ETL/Ingestor/pdoIngestor.php +++ b/classes/ETL/Ingestor/pdoIngestor.php @@ -605,7 +605,7 @@ private function singleDatabaseIngest() $updateColumnList = array_map(function($s) { return "$s=VALUES($s)"; }, $destColumnList); $updateColumns = implode(',', $updateColumnList); $sql = "INSERT INTO $qualifiedDestTableName ($destColumns) " . $this->sourceQueryString - . " ON DUPLICATE KEY UPDATE $updateColumns"; + . "\nON DUPLICATE KEY UPDATE $updateColumns"; } $this->logger->debug($sql); From 7b321424e035f7aa3f55e679cdc89acbd6e3268c Mon Sep 17 00:00:00 2001 From: Steve Gallo Date: Fri, 20 Jan 2017 13:22:41 -0500 Subject: [PATCH 06/15] Add modern PDO support for Oracle (PDOOCI wraps the OCI8 library) --- composer.json | 3 +- composer.lock | 442 ++++++++++++++++++++------------------------------ 2 files changed, 178 insertions(+), 267 deletions(-) diff --git a/composer.json b/composer.json index 071f049bff..bf6900de9c 100644 --- a/composer.json +++ b/composer.json @@ -22,7 +22,8 @@ "apache/commons-collections": "^2.1.1", "apache/commons-digester": "^1.7", "apache/commons-logging": "^1.0.4", - "egulias/email-validator": "^1.2" + "egulias/email-validator": "^1.2", + "taq/pdooci": "^1.0" }, "require-dev": { "phpunit/phpunit": "4.8.*", diff --git a/composer.lock b/composer.lock index 9850e58b72..0c6810754e 100644 --- a/composer.lock +++ b/composer.lock @@ -4,8 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", "This file is @generated automatically" ], - "hash": "687a71617f008a0ba19058cff76f8abf", - "content-hash": "6fdb530ce2e952c4d5da1f0e3a9a1a09", + "content-hash": "56d4332d0a9b08011dbc37e7a409449d", "packages": [ { "name": "apache/commons-beanutils", @@ -259,7 +258,7 @@ "zend", "zikula" ], - "time": "2016-08-13 20:53:52" + "time": "2016-08-13T20:53:52+00:00" }, { "name": "doctrine/lexer", @@ -313,7 +312,7 @@ "lexer", "parser" ], - "time": "2014-09-09 13:34:57" + "time": "2014-09-09T13:34:57+00:00" }, { "name": "egulias/email-validator", @@ -365,7 +364,7 @@ "validation", "validator" ], - "time": "2016-07-03 21:52:18" + "time": "2016-07-03T21:52:18+00:00" }, { "name": "greenlion/php-sql-parser", @@ -421,7 +420,7 @@ "parser", "sql" ], - "time": "2014-04-16 10:03:29" + "time": "2014-04-16T10:03:29+00:00" }, { "name": "highsoft/highcharts", @@ -504,23 +503,23 @@ }, { "name": "justinrainbow/json-schema", - "version": "2.0.3", + "version": "2.0.5", "source": { "type": "git", "url": "https://github.com/justinrainbow/json-schema.git", - "reference": "c21534c635f03428e92254333fab4ae35b2cdfd9" + "reference": "6b2a33e6a768f96bdc2ead5600af0822eed17d67" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/justinrainbow/json-schema/zipball/c21534c635f03428e92254333fab4ae35b2cdfd9", - "reference": "c21534c635f03428e92254333fab4ae35b2cdfd9", + "url": "https://api.github.com/repos/justinrainbow/json-schema/zipball/6b2a33e6a768f96bdc2ead5600af0822eed17d67", + "reference": "6b2a33e6a768f96bdc2ead5600af0822eed17d67", "shasum": "" }, "require": { "php": ">=5.3.3" }, "require-dev": { - "json-schema/json-schema-test-suite": "1.1.2", + "json-schema/json-schema-test-suite": "1.2.0", "phpdocumentor/phpdocumentor": "~2", "phpunit/phpunit": "^4.8.22" }, @@ -566,7 +565,7 @@ "json", "schema" ], - "time": "2016-05-10 20:38:51" + "time": "2016-06-02T10:59:52+00:00" }, { "name": "moment/moment-min-file", @@ -703,26 +702,34 @@ "container", "dependency injection" ], - "time": "2013-11-22 08:30:29" + "time": "2013-11-22T08:30:29+00:00" }, { "name": "psr/log", - "version": "1.0.0", + "version": "1.0.2", "source": { "type": "git", "url": "https://github.com/php-fig/log.git", - "reference": "fe0936ee26643249e916849d48e3a51d5f5e278b" + "reference": "4ebe3a8bf773a19edfe0a84b6585ba3d401b724d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-fig/log/zipball/fe0936ee26643249e916849d48e3a51d5f5e278b", - "reference": "fe0936ee26643249e916849d48e3a51d5f5e278b", + "url": "https://api.github.com/repos/php-fig/log/zipball/4ebe3a8bf773a19edfe0a84b6585ba3d401b724d", + "reference": "4ebe3a8bf773a19edfe0a84b6585ba3d401b724d", "shasum": "" }, + "require": { + "php": ">=5.3.0" + }, "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, "autoload": { - "psr-0": { - "Psr\\Log\\": "" + "psr-4": { + "Psr\\Log\\": "Psr/Log/" } }, "notification-url": "https://packagist.org/downloads/", @@ -736,25 +743,26 @@ } ], "description": "Common interface for logging libraries", + "homepage": "https://github.com/php-fig/log", "keywords": [ "log", "psr", "psr-3" ], - "time": "2012-12-21 11:40:51" + "time": "2016-10-10T12:19:37+00:00" }, { "name": "robrichards/xmlseclibs", - "version": "1.4.1", + "version": "1.4.2", "source": { "type": "git", "url": "https://github.com/robrichards/xmlseclibs.git", - "reference": "465f18a8e1196c279b1298a3b08bcbee71ea4e4e" + "reference": "79fb5e03c4ee4dc3ec77e4b2628231374364a017" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/robrichards/xmlseclibs/zipball/465f18a8e1196c279b1298a3b08bcbee71ea4e4e", - "reference": "465f18a8e1196c279b1298a3b08bcbee71ea4e4e", + "url": "https://api.github.com/repos/robrichards/xmlseclibs/zipball/79fb5e03c4ee4dc3ec77e4b2628231374364a017", + "reference": "79fb5e03c4ee4dc3ec77e4b2628231374364a017", "shasum": "" }, "require": { @@ -782,7 +790,7 @@ "xml", "xmldsig" ], - "time": "2015-07-31 12:22:14" + "time": "2016-09-08T13:31:44+00:00" }, { "name": "sencha/extjs-gpl", @@ -885,20 +893,20 @@ "keywords": [ "microframework" ], - "time": "2015-06-04 21:24:58" + "time": "2015-06-04T21:24:58+00:00" }, { "name": "simplesamlphp/saml2", - "version": "v1.8", + "version": "v1.9.1", "source": { "type": "git", "url": "https://github.com/simplesamlphp/saml2.git", - "reference": "11891064b8a2d1a483925cade7a9277f36b6055e" + "reference": "2a2bd4398257cbe72251c68348be72707cc77262" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/simplesamlphp/saml2/zipball/11891064b8a2d1a483925cade7a9277f36b6055e", - "reference": "11891064b8a2d1a483925cade7a9277f36b6055e", + "url": "https://api.github.com/repos/simplesamlphp/saml2/zipball/2a2bd4398257cbe72251c68348be72707cc77262", + "reference": "2a2bd4398257cbe72251c68348be72707cc77262", "shasum": "" }, "require": { @@ -934,26 +942,34 @@ } ], "description": "SAML2 PHP library from SimpleSAMLphp", - "time": "2016-01-27 14:23:34" + "time": "2016-12-02T12:45:13+00:00" }, { "name": "simplesamlphp/simplesamlphp", - "version": "v1.14.0", + "version": "v1.14.11", "source": { "type": "git", "url": "https://github.com/simplesamlphp/simplesamlphp.git", - "reference": "03c6303dfee814450836c4ee3d07d51e628e4d4e" + "reference": "b96dcc500caae73954d4f01189e0209afc6086be" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/simplesamlphp/simplesamlphp/zipball/03c6303dfee814450836c4ee3d07d51e628e4d4e", - "reference": "03c6303dfee814450836c4ee3d07d51e628e4d4e", + "url": "https://api.github.com/repos/simplesamlphp/simplesamlphp/zipball/b96dcc500caae73954d4f01189e0209afc6086be", + "reference": "b96dcc500caae73954d4f01189e0209afc6086be", "shasum": "" }, "require": { + "ext-date": "*", + "ext-dom": "*", + "ext-hash": "*", + "ext-json": "*", + "ext-openssl": "*", + "ext-pcre": "*", + "ext-spl": "*", + "ext-zlib": "*", "php": ">=5.3", "robrichards/xmlseclibs": "~1.4.1", - "simplesamlphp/saml2": "~1.7", + "simplesamlphp/saml2": "~1.9.1", "whitehat101/apr1-md5": "~1.0" }, "require-dev": { @@ -996,7 +1012,7 @@ "sp", "ws-federation" ], - "time": "2016-02-15 12:18:12" + "time": "2016-12-12T15:13:53+00:00" }, { "name": "symfony/debug", @@ -1057,7 +1073,7 @@ ], "description": "Symfony Debug Component", "homepage": "https://symfony.com", - "time": "2015-07-08 05:59:48" + "time": "2015-07-08T05:59:48+00:00" }, { "name": "symfony/event-dispatcher", @@ -1116,7 +1132,7 @@ ], "description": "Symfony EventDispatcher Component", "homepage": "https://symfony.com", - "time": "2015-05-02 15:18:45" + "time": "2015-05-02T15:18:45+00:00" }, { "name": "symfony/http-foundation", @@ -1170,7 +1186,7 @@ ], "description": "Symfony HttpFoundation Component", "homepage": "https://symfony.com", - "time": "2015-07-22 10:08:40" + "time": "2015-07-22T10:08:40+00:00" }, { "name": "symfony/http-kernel", @@ -1248,7 +1264,7 @@ ], "description": "Symfony HttpKernel Component", "homepage": "https://symfony.com", - "time": "2016-01-14 10:11:16" + "time": "2016-01-14T10:11:16+00:00" }, { "name": "symfony/process", @@ -1298,7 +1314,7 @@ ], "description": "Symfony Process Component", "homepage": "https://symfony.com", - "time": "2015-06-30 16:10:16" + "time": "2015-06-30T16:10:16+00:00" }, { "name": "symfony/routing", @@ -1367,7 +1383,52 @@ "uri", "url" ], - "time": "2015-07-09 16:02:48" + "time": "2015-07-09T16:02:48+00:00" + }, + { + "name": "taq/pdooci", + "version": "1.0.7", + "source": { + "type": "git", + "url": "https://github.com/taq/pdooci.git", + "reference": "898a8fcef39b7cf748454c06e98c45baecffca94" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/taq/pdooci/zipball/898a8fcef39b7cf748454c06e98c45baecffca94", + "reference": "898a8fcef39b7cf748454c06e98c45baecffca94", + "shasum": "" + }, + "require": { + "php": ">=5.3.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "PDOOCI\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "GPL" + ], + "authors": [ + { + "name": "Eustaquio Rangel", + "email": "eustaquiorangel@gmail.com", + "homepage": "http://eustaquiorangel.com", + "role": "Developer" + } + ], + "description": "Replacement for the PHP PDO OCI class", + "homepage": "http://github.com/taq/pdooci", + "keywords": [ + "oci", + "oracle", + "pdo", + "pdo_oci" + ], + "time": "2016-09-24T21:51:27+00:00" }, { "name": "tildeio/rsvpjs-min-file", @@ -1432,7 +1493,7 @@ "MD5", "apr1" ], - "time": "2015-02-11 11:06:42" + "time": "2015-02-11T11:06:42+00:00" }, { "name": "zendframework/zendframework-minimal", @@ -1509,49 +1570,7 @@ "constructor", "instantiate" ], - "time": "2015-06-14 21:17:01" - }, - { - "name": "ircmaxell/password-compat", - "version": "v1.0.4", - "source": { - "type": "git", - "url": "https://github.com/ircmaxell/password_compat.git", - "reference": "5c5cde8822a69545767f7c7f3058cb15ff84614c" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/ircmaxell/password_compat/zipball/5c5cde8822a69545767f7c7f3058cb15ff84614c", - "reference": "5c5cde8822a69545767f7c7f3058cb15ff84614c", - "shasum": "" - }, - "require-dev": { - "phpunit/phpunit": "4.*" - }, - "type": "library", - "autoload": { - "files": [ - "lib/password.php" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Anthony Ferrara", - "email": "ircmaxell@php.net", - "homepage": "http://blog.ircmaxell.com" - } - ], - "description": "A compatibility library for the proposed simplified password hashing algorithm: https://wiki.php.net/rfc/password_hash", - "homepage": "https://github.com/ircmaxell/password_compat", - "keywords": [ - "hashing", - "password" - ], - "time": "2014-11-20 16:49:30" + "time": "2015-06-14T21:17:01+00:00" }, { "name": "phpdocumentor/reflection-docblock", @@ -1600,36 +1619,37 @@ "email": "mike.vanriel@naenius.com" } ], - "time": "2015-02-03 12:10:50" + "time": "2015-02-03T12:10:50+00:00" }, { "name": "phpspec/prophecy", - "version": "v1.6.0", + "version": "v1.6.2", "source": { "type": "git", "url": "https://github.com/phpspec/prophecy.git", - "reference": "3c91bdf81797d725b14cb62906f9a4ce44235972" + "reference": "6c52c2722f8460122f96f86346600e1077ce22cb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpspec/prophecy/zipball/3c91bdf81797d725b14cb62906f9a4ce44235972", - "reference": "3c91bdf81797d725b14cb62906f9a4ce44235972", + "url": "https://api.github.com/repos/phpspec/prophecy/zipball/6c52c2722f8460122f96f86346600e1077ce22cb", + "reference": "6c52c2722f8460122f96f86346600e1077ce22cb", "shasum": "" }, "require": { "doctrine/instantiator": "^1.0.2", "php": "^5.3|^7.0", - "phpdocumentor/reflection-docblock": "~2.0", - "sebastian/comparator": "~1.1", - "sebastian/recursion-context": "~1.0" + "phpdocumentor/reflection-docblock": "^2.0|^3.0.2", + "sebastian/comparator": "^1.1", + "sebastian/recursion-context": "^1.0|^2.0" }, "require-dev": { - "phpspec/phpspec": "~2.0" + "phpspec/phpspec": "^2.0", + "phpunit/phpunit": "^4.8 || ^5.6.5" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "1.5.x-dev" + "dev-master": "1.6.x-dev" } }, "autoload": { @@ -1662,7 +1682,7 @@ "spy", "stub" ], - "time": "2016-02-15 07:46:21" + "time": "2016-11-21T14:58:47+00:00" }, { "name": "phpunit/php-code-coverage", @@ -1724,20 +1744,20 @@ "testing", "xunit" ], - "time": "2015-10-06 15:47:00" + "time": "2015-10-06T15:47:00+00:00" }, { "name": "phpunit/php-file-iterator", - "version": "1.4.1", + "version": "1.4.2", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-file-iterator.git", - "reference": "6150bf2c35d3fc379e50c7602b75caceaa39dbf0" + "reference": "3cc8f69b3028d0f96a9078e6295d86e9bf019be5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/6150bf2c35d3fc379e50c7602b75caceaa39dbf0", - "reference": "6150bf2c35d3fc379e50c7602b75caceaa39dbf0", + "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/3cc8f69b3028d0f96a9078e6295d86e9bf019be5", + "reference": "3cc8f69b3028d0f96a9078e6295d86e9bf019be5", "shasum": "" }, "require": { @@ -1771,7 +1791,7 @@ "filesystem", "iterator" ], - "time": "2015-06-21 13:08:43" + "time": "2016-10-03T07:40:28+00:00" }, { "name": "phpunit/php-text-template", @@ -1812,25 +1832,28 @@ "keywords": [ "template" ], - "time": "2015-06-21 13:50:34" + "time": "2015-06-21T13:50:34+00:00" }, { "name": "phpunit/php-timer", - "version": "1.0.7", + "version": "1.0.8", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-timer.git", - "reference": "3e82f4e9fc92665fafd9157568e4dcb01d014e5b" + "reference": "38e9124049cf1a164f1e4537caf19c99bf1eb260" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-timer/zipball/3e82f4e9fc92665fafd9157568e4dcb01d014e5b", - "reference": "3e82f4e9fc92665fafd9157568e4dcb01d014e5b", + "url": "https://api.github.com/repos/sebastianbergmann/php-timer/zipball/38e9124049cf1a164f1e4537caf19c99bf1eb260", + "reference": "38e9124049cf1a164f1e4537caf19c99bf1eb260", "shasum": "" }, "require": { "php": ">=5.3.3" }, + "require-dev": { + "phpunit/phpunit": "~4|~5" + }, "type": "library", "autoload": { "classmap": [ @@ -1853,20 +1876,20 @@ "keywords": [ "timer" ], - "time": "2015-06-21 08:01:12" + "time": "2016-05-12T18:03:57+00:00" }, { "name": "phpunit/php-token-stream", - "version": "1.4.8", + "version": "1.4.9", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-token-stream.git", - "reference": "3144ae21711fb6cac0b1ab4cbe63b75ce3d4e8da" + "reference": "3b402f65a4cc90abf6e1104e388b896ce209631b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-token-stream/zipball/3144ae21711fb6cac0b1ab4cbe63b75ce3d4e8da", - "reference": "3144ae21711fb6cac0b1ab4cbe63b75ce3d4e8da", + "url": "https://api.github.com/repos/sebastianbergmann/php-token-stream/zipball/3b402f65a4cc90abf6e1104e388b896ce209631b", + "reference": "3b402f65a4cc90abf6e1104e388b896ce209631b", "shasum": "" }, "require": { @@ -1902,20 +1925,20 @@ "keywords": [ "tokenizer" ], - "time": "2015-09-15 10:49:45" + "time": "2016-11-15T14:06:22+00:00" }, { "name": "phpunit/phpunit", - "version": "4.8.23", + "version": "4.8.31", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "6e351261f9cd33daf205a131a1ba61c6d33bd483" + "reference": "98b2b39a520766bec663ff5b7ff1b729db9dbfe3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/6e351261f9cd33daf205a131a1ba61c6d33bd483", - "reference": "6e351261f9cd33daf205a131a1ba61c6d33bd483", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/98b2b39a520766bec663ff5b7ff1b729db9dbfe3", + "reference": "98b2b39a520766bec663ff5b7ff1b729db9dbfe3", "shasum": "" }, "require": { @@ -1929,9 +1952,9 @@ "phpunit/php-code-coverage": "~2.1", "phpunit/php-file-iterator": "~1.4", "phpunit/php-text-template": "~1.2", - "phpunit/php-timer": ">=1.0.6", + "phpunit/php-timer": "^1.0.6", "phpunit/phpunit-mock-objects": "~2.3", - "sebastian/comparator": "~1.1", + "sebastian/comparator": "~1.2.2", "sebastian/diff": "~1.2", "sebastian/environment": "~1.3", "sebastian/exporter": "~1.2", @@ -1974,7 +1997,7 @@ "testing", "xunit" ], - "time": "2016-02-11 14:56:33" + "time": "2016-12-09T02:45:31+00:00" }, { "name": "phpunit/phpunit-mock-objects", @@ -2030,26 +2053,26 @@ "mock", "xunit" ], - "time": "2015-10-02 06:51:40" + "time": "2015-10-02T06:51:40+00:00" }, { "name": "sebastian/comparator", - "version": "1.2.0", + "version": "1.2.2", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/comparator.git", - "reference": "937efb279bd37a375bcadf584dec0726f84dbf22" + "reference": "6a1ed12e8b2409076ab22e3897126211ff8b1f7f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/937efb279bd37a375bcadf584dec0726f84dbf22", - "reference": "937efb279bd37a375bcadf584dec0726f84dbf22", + "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/6a1ed12e8b2409076ab22e3897126211ff8b1f7f", + "reference": "6a1ed12e8b2409076ab22e3897126211ff8b1f7f", "shasum": "" }, "require": { "php": ">=5.3.3", "sebastian/diff": "~1.2", - "sebastian/exporter": "~1.2" + "sebastian/exporter": "~1.2 || ~2.0" }, "require-dev": { "phpunit/phpunit": "~4.4" @@ -2094,7 +2117,7 @@ "compare", "equality" ], - "time": "2015-07-26 15:48:44" + "time": "2016-11-19T09:18:40+00:00" }, { "name": "sebastian/diff", @@ -2146,27 +2169,27 @@ "keywords": [ "diff" ], - "time": "2015-12-08 07:14:41" + "time": "2015-12-08T07:14:41+00:00" }, { "name": "sebastian/environment", - "version": "1.3.3", + "version": "1.3.8", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/environment.git", - "reference": "6e7133793a8e5a5714a551a8324337374be209df" + "reference": "be2c607e43ce4c89ecd60e75c6a85c126e754aea" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/6e7133793a8e5a5714a551a8324337374be209df", - "reference": "6e7133793a8e5a5714a551a8324337374be209df", + "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/be2c607e43ce4c89ecd60e75c6a85c126e754aea", + "reference": "be2c607e43ce4c89ecd60e75c6a85c126e754aea", "shasum": "" }, "require": { - "php": ">=5.3.3" + "php": "^5.3.3 || ^7.0" }, "require-dev": { - "phpunit/phpunit": "~4.4" + "phpunit/phpunit": "^4.8 || ^5.0" }, "type": "library", "extra": { @@ -2196,20 +2219,20 @@ "environment", "hhvm" ], - "time": "2015-12-02 08:37:27" + "time": "2016-08-18T05:49:44+00:00" }, { "name": "sebastian/exporter", - "version": "1.2.1", + "version": "1.2.2", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/exporter.git", - "reference": "7ae5513327cb536431847bcc0c10edba2701064e" + "reference": "42c4c2eec485ee3e159ec9884f95b431287edde4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/7ae5513327cb536431847bcc0c10edba2701064e", - "reference": "7ae5513327cb536431847bcc0c10edba2701064e", + "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/42c4c2eec485ee3e159ec9884f95b431287edde4", + "reference": "42c4c2eec485ee3e159ec9884f95b431287edde4", "shasum": "" }, "require": { @@ -2217,12 +2240,13 @@ "sebastian/recursion-context": "~1.0" }, "require-dev": { + "ext-mbstring": "*", "phpunit/phpunit": "~4.4" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "1.2.x-dev" + "dev-master": "1.3.x-dev" } }, "autoload": { @@ -2262,7 +2286,7 @@ "export", "exporter" ], - "time": "2015-06-21 07:55:53" + "time": "2016-06-17T09:04:28+00:00" }, { "name": "sebastian/global-state", @@ -2313,7 +2337,7 @@ "keywords": [ "global state" ], - "time": "2015-10-12 03:26:01" + "time": "2015-10-12T03:26:01+00:00" }, { "name": "sebastian/recursion-context", @@ -2366,7 +2390,7 @@ ], "description": "Provides functionality to recursively process PHP variables", "homepage": "http://www.github.com/sebastianbergmann/recursion-context", - "time": "2015-11-11 19:50:13" + "time": "2015-11-11T19:50:13+00:00" }, { "name": "sebastian/version", @@ -2401,20 +2425,20 @@ ], "description": "Library that helps with managing the version number of Git-hosted PHP projects", "homepage": "https://github.com/sebastianbergmann/version", - "time": "2015-06-21 13:59:46" + "time": "2015-06-21T13:59:46+00:00" }, { "name": "squizlabs/php_codesniffer", - "version": "2.6.0", + "version": "2.7.1", "source": { "type": "git", "url": "https://github.com/squizlabs/PHP_CodeSniffer.git", - "reference": "1bcdf03b068a530ac1962ce671dead356eeba43b" + "reference": "9b324f3a1132459a7274a0ace2e1b766ba80930f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/1bcdf03b068a530ac1962ce671dead356eeba43b", - "reference": "1bcdf03b068a530ac1962ce671dead356eeba43b", + "url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/9b324f3a1132459a7274a0ace2e1b766ba80930f", + "reference": "9b324f3a1132459a7274a0ace2e1b766ba80930f", "shasum": "" }, "require": { @@ -2479,121 +2503,7 @@ "phpcs", "standards" ], - "time": "2016-04-03 22:58:34" - }, - { - "name": "symfony/polyfill-php54", - "version": "v1.1.0", - "source": { - "type": "git", - "url": "https://github.com/symfony/polyfill-php54.git", - "reference": "74663d5a2ff3c530c1bc0571500e0feec9094054" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php54/zipball/74663d5a2ff3c530c1bc0571500e0feec9094054", - "reference": "74663d5a2ff3c530c1bc0571500e0feec9094054", - "shasum": "" - }, - "require": { - "php": ">=5.3.3" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.1-dev" - } - }, - "autoload": { - "psr-4": { - "Symfony\\Polyfill\\Php54\\": "" - }, - "files": [ - "bootstrap.php" - ], - "classmap": [ - "Resources/stubs" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Nicolas Grekas", - "email": "p@tchwork.com" - }, - { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" - } - ], - "description": "Symfony polyfill backporting some PHP 5.4+ features to lower PHP versions", - "homepage": "https://symfony.com", - "keywords": [ - "compatibility", - "polyfill", - "portable", - "shim" - ], - "time": "2016-01-20 09:13:37" - }, - { - "name": "symfony/polyfill-php55", - "version": "v1.1.0", - "source": { - "type": "git", - "url": "https://github.com/symfony/polyfill-php55.git", - "reference": "b4f3f07d91702f8f926339fc4fcf81671d8c27e6" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php55/zipball/b4f3f07d91702f8f926339fc4fcf81671d8c27e6", - "reference": "b4f3f07d91702f8f926339fc4fcf81671d8c27e6", - "shasum": "" - }, - "require": { - "ircmaxell/password-compat": "~1.0", - "php": ">=5.3.3" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.1-dev" - } - }, - "autoload": { - "psr-4": { - "Symfony\\Polyfill\\Php55\\": "" - }, - "files": [ - "bootstrap.php" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Nicolas Grekas", - "email": "p@tchwork.com" - }, - { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" - } - ], - "description": "Symfony polyfill backporting some PHP 5.5+ features to lower PHP versions", - "homepage": "https://symfony.com", - "keywords": [ - "compatibility", - "polyfill", - "portable", - "shim" - ], - "time": "2016-01-20 09:13:37" + "time": "2016-11-30T04:02:31+00:00" }, { "name": "symfony/yaml", @@ -2643,7 +2553,7 @@ ], "description": "Symfony Yaml Component", "homepage": "https://symfony.com", - "time": "2015-07-26 08:59:42" + "time": "2015-07-26T08:59:42+00:00" } ], "aliases": [], From c6cef7e27a80f6f64555db7a42ecb7d75b4def67 Mon Sep 17 00:00:00 2001 From: Steve Gallo Date: Fri, 20 Jan 2017 13:23:38 -0500 Subject: [PATCH 07/15] Support for Oracle ETL data endpoint (readonly) --- classes/CCR/DB/OracleDB.php | 136 ++++++++++++-- classes/CCR/DB/PDODB.php | 67 ++++--- classes/ETL/DataEndpoint.php | 6 +- classes/ETL/DataEndpoint/Oracle.php | 190 +++++++++++++++++++ classes/ETL/DataEndpoint/README_ORACLE.md | 219 ++++++++++++++++++++++ 5 files changed, 575 insertions(+), 43 deletions(-) create mode 100644 classes/ETL/DataEndpoint/Oracle.php create mode 100644 classes/ETL/DataEndpoint/README_ORACLE.md diff --git a/classes/CCR/DB/OracleDB.php b/classes/CCR/DB/OracleDB.php index 1f605f530f..6d42a3fafa 100644 --- a/classes/CCR/DB/OracleDB.php +++ b/classes/CCR/DB/OracleDB.php @@ -1,22 +1,124 @@ + /* ------------------------------------------------------------------------------------------ + * Set up the machinery. Oracle requires at minimum a database name (local naming, + * this resolves to an entry in tnsnames.org) or a name, host, and optionally a port + * (easy connect naming). + * + * Note: We do not support the $dsn_override that PDODB supports. + * + * @see PDODB::__construct() + * + * @throw Exception If minimum parameters were not provided. + * + * @see iDatabase::__construct() + * ------------------------------------------------------------------------------------------ + */ + + public function __construct($db_host, $db_port, $db_name, $db_username, $db_password, $dsn_extra = null) + { + // At a minimum we must have either the (db_name) or the (db_host, db_name) + + if ( null === $db_name ) { + $msg = __CLASS__ + . ' requires at a minimum (db_name) for Local Naming ' + . ' or (db_host, db_name[, db_port]) for Easy Connect Naming'; + throw new Exception($msg); + } + + parent::__construct( + "oci", + $db_host, + $db_port, + $db_name, + $db_username, + $db_password + ); + + } // __construct() + + /* ------------------------------------------------------------------------------------------ + * Clean up after ourselves and close the connection. Unlike standard PDO setting the + * connection to NULL OCI has an oci_close() function. + * ------------------------------------------------------------------------------------------ + */ + + public function __destruct() + { + if ( null !== $this->_dbh) { + $this->_dbh->close(); + } + } // __destruct() + + /* ------------------------------------------------------------------------------------------ + * Connect to the server. + * + * @return The database connection handle + * + * @throw Exception If there was a connection error + * ------------------------------------------------------------------------------------------ + */ + + public function connect() + { + if ( null !== $this->_dbh ) { + return $this->_dbh; + } + + $namingMethod = null; + + // If the host is not set then assume we are using Easy Connect + // https://docs.oracle.com/database/121/NETAG/naming.htm#NETAG008 + // For example oci:dbname=//localhost:1521/mydb + // + // Otherwise assume Local Naming (via tnsnames.ora) + // https://docs.oracle.com/database/121/NETAG/naming.htm#NETAG081 + // For example oci:dbname=mydb + + if ( null === $this->_db_host ) { + $this->_dsn = 'oci:dbname=' . $this->_db_name; + $namingMethod = "Local"; + } else { + $this->_dsn = 'oci:dbname=//' + . $this->_db_host + . ( null !== $this->_db_port ? ':' . $this->_db_port : '' ) + . '/' . $this->_db_name; + $namingMethod = "Easy Connect"; + } + + + try { + $this->_dbh = @new \PDOOCI\PDO($this->_dsn, $this->_db_username, $this->_db_password); + $this->_dbh->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); + + } catch (\PDOException $e) { + $msg = __CLASS__ + . " Error connecting to database '" . $this->_dsn . "' using $namingMethod Naming. " + . $e->getMessage(); + throw new Exception($msg); + } + + return $this->_dbh; + + } // connect() +} // class OracleDB diff --git a/classes/CCR/DB/PDODB.php b/classes/CCR/DB/PDODB.php index 92575ba151..ad9127d7dd 100644 --- a/classes/CCR/DB/PDODB.php +++ b/classes/CCR/DB/PDODB.php @@ -25,23 +25,25 @@ use PDO; use Exception; -class PDODB -implements iDatabase +class PDODB implements iDatabase { // Database connection parameters - public $_db_engine = NULL; - public $_db_host = NULL; - public $_db_port = NULL; - public $_db_name = NULL; - public $_db_username = NULL; - public $_db_password = NULL; + public $_db_engine = null; + public $_db_host = null; + public $_db_port = null; + public $_db_name = null; + public $_db_username = null; + public $_db_password = null; // Optional extra parameters to be added to the DSN - public $_dsn_extra = NULL; + public $_dsn_extra = null; + + // Generated DSN + protected $_dsn = null; // Handle to the PDO instance - protected $_dbh = NULL; + protected $_dbh = null; protected static $_debug_mode = false; protected static $_queries = array(); @@ -81,8 +83,8 @@ public function connect() return $this->_dbh; } - $dsn = $this->generateDsn(); - $this->_dbh = new PDO($dsn, $this->_db_username, $this->_db_password); + $this->_dsn = $this->generateDsn(); + $this->_dbh = new PDO($this->_dsn, $this->_db_username, $this->_db_password); $this->_dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); return $this->_dbh; @@ -94,7 +96,7 @@ public function connect() public function disconnect() { - $this->_dbh = NULL; + $this->_dbh = null; } @@ -127,6 +129,15 @@ protected function generateDsn() return $dsn; } // generateDsn() + // ------------------------------------------------------------------------------------------ + // @return The generated DSN, or NULL of no DSN has been generated. + // ------------------------------------------------------------------------------------------ + + public function getDsn() + { + return $this->_dsn; + } // getDsn() + // -------------------------------------------------------------------------------- // @see iDatabase::query() // -------------------------------------------------------------------------------- @@ -136,12 +147,14 @@ public function query($query, array $params = array(), $returnStatement = false) $stmt = $this->prepare($query); try { - if ( $this->debugging() ) $this->debug($query, $params); + if ( $this->debugging() ) { + $this->debug($query, $params); + } } catch (Exception $e) { // TODO: setup the logger and log this. } - if (FALSE === $stmt->execute($params)) { + if (false === $stmt->execute($params)) { list($sqlState, $errorCode, $errorMsg) = $stmt->errorInfo; throw new Exception("$sqlState: $errorMsg ($errorCode)"); } @@ -161,12 +174,14 @@ public function execute($query, array $params = array()) $stmt = $this->prepare($query); try { - if ( $this->debugging() ) $this->debug($query, $params); + if ( $this->debugging() ) { + $this->debug($query, $params); + } } catch (Exception $e) { // TODO: setup the logger and log this. } - if (FALSE === $stmt->execute($params)) { + if (false === $stmt->execute($params)) { list($sqlState, $errorCode, $errorMsg) = $stmt->errorInfo; throw new Exception("$sqlState: $errorMsg ($errorCode)"); } @@ -198,12 +213,14 @@ public function insert($statement, $params = array()) $stmt = $this->prepare($statement); try { - if ( $this->debugging() ) $this->debug($statement, $params); + if ( $this->debugging() ) { + $this->debug($statement, $params); + } } catch (Exception $e) { // TODO: setup the logger and log this. } - if (FALSE === $stmt->execute($params)) { + if (false === $stmt->execute($params)) { list($sqlState, $errorCode, $errorMsg) = $stmt->errorInfo; throw new Exception("$sqlState: $errorMsg ($errorCode)"); } @@ -226,7 +243,9 @@ public function getRowCount($schema, $table) $query = "select count(*) as count_result from $full_tablename"; try { - if ( $this->debugging() ) $this->debug($query, array()); + if ( $this->debugging() ) { + $this->debug($query, array()); + } } catch (Exception $e) { // TODO: setup the logger and log this. } @@ -312,15 +331,15 @@ private function debugging() try { $sql_debug_mode = \xd_utilities\getConfiguration('general', 'sql_debug_mode'); - } catch (Exception $e) {} - + } catch (Exception $e) { + } return $sql_debug_mode || PDODB::$_debug_mode; } private function debug($query, $params) { - PDODB::$_queries[] = trim(preg_replace("(\s+)"," ", $query)); + PDODB::$_queries[] = trim(preg_replace("(\s+)", " ", $query)); PDODB::$_params[] = PDODB::protectParams($params); } @@ -335,4 +354,4 @@ private static function protectParams($params) } return $params; } -} +} // class PDODB diff --git a/classes/ETL/DataEndpoint.php b/classes/ETL/DataEndpoint.php index a9e26b9454..1f42378465 100644 --- a/classes/ETL/DataEndpoint.php +++ b/classes/ETL/DataEndpoint.php @@ -26,12 +26,14 @@ class DataEndpoint const TYPE_MYSQL = "mysql"; const TYPE_POSTGRES = "postgres"; + const TYPE_ORACLE = "oracle"; const TYPE_FILE = "file"; const TYPE_JSONFILE = "jsonfile"; const TYPE_REST = "rest"; private static $supportedTypes = array(self::TYPE_MYSQL, + self::TYPE_ORACLE, self::TYPE_POSTGRES, self::TYPE_FILE, self::TYPE_JSONFILE, @@ -40,6 +42,7 @@ class DataEndpoint private static $classmap = array( self::TYPE_MYSQL => 'ETL\DataEndpoint\Mysql', self::TYPE_POSTGRES => 'ETL\DataEndpoint\Postgres', + self::TYPE_ORACLE => 'ETL\DataEndpoint\Oracle', self::TYPE_FILE => 'ETL\DataEndpoint\File', self::TYPE_JSONFILE => 'ETL\DataEndpoint\JsonFile', self::TYPE_REST => 'ETL\DataEndpoint\Rest' @@ -97,5 +100,4 @@ public static function factory(DataEndpointOptions $options, Log $logger = null) return $endpoint; } // factory() - -} // class DataEndpoint \ No newline at end of file +} // class DataEndpoint diff --git a/classes/ETL/DataEndpoint/Oracle.php b/classes/ETL/DataEndpoint/Oracle.php new file mode 100644 index 0000000000..afd5fa2124 --- /dev/null +++ b/classes/ETL/DataEndpoint/Oracle.php @@ -0,0 +1,190 @@ + + * @data 2017-01-13 + * ========================================================================================== + */ + +namespace ETL\DataEndpoint; + +use ETL\DataEndpoint\DataEndpointOptions; +use \Log; + +class Oracle extends aRdbmsEndpoint implements iRdbmsEndpoint +{ + + public function __construct(DataEndpointOptions $options, Log $logger = null) + { + parent::__construct($options, $logger); + $this->systemQuoteChar = '"'; + } // __construct() + + /* ------------------------------------------------------------------------------------------ + * We consider 2 Postgres servers to be the same if the host and port are equal. Query both the + * current and comparison endpoints and compare. + * + * @see iDataEndpoint::isSameServer() + * ------------------------------------------------------------------------------------------ + */ + + public function isSameServer(iDataEndpoint $cmp) + { + return false; + } // isSameServer() + + /* ------------------------------------------------------------------------------------------ + * @see iRdbmsEndpoint::schemaExists() + * ------------------------------------------------------------------------------------------ + */ + + public function schemaExists($schemaName = null) + { + if ( null === $schemaName ) { + $schemaName = $this->getSchema(); + } elseif ( empty($schemaName) ) { + $msg = "Schema name cannot be empty"; + $this->logAndThrowException($msg); + } + + // See http://www.postgresql.org/docs/current/static/catalogs.html + + $sql = "SELECT +username AS name +FROM all_users +WHERE username = UPPER(:schema)"; + + $params = array(":schema" => $this->getSchema()); + + try { + $dbh = $this->getHandle(); + $result = $dbh->query($sql, $params); + if ( 0 == count($result) ) { + return false; + } + } catch (\PdoException $e) { + $msg = "Error querying for schema '" . $this->getSchema() . "'"; + $this->logAndThrowSqlException($sql, $e, $msg); + } + + return true; + + } // schemaExists() + + /* ------------------------------------------------------------------------------------------ + * @see iRdbmsEndpoint::createSchema() + * ------------------------------------------------------------------------------------------ + */ + + public function createSchema($schemaName = null) + { + if ( null === $schemaName ) { + $schemaName = $this->getSchema(true); + } elseif ( empty($schemaName) ) { + $msg = "Schema name cannot be empty"; + $this->logAndThrowException($msg); + } else { + $schemaName = ( 0 !== strpos($schemaName, $this->systemQuoteChar) + ? $this->quoteSystemIdentifier($schemaName) + : $schemaName ); + } + + // Creating Oracle schemas require creating a user including specifying an + // authentication method. Since we do not support writing to Oracle at the moment, + // we don't need to support schema creation. + + return false; + + } // createSchema() + + /* ------------------------------------------------------------------------------------------ + * @see iRdbmsEndpoint::tableExists() + * ------------------------------------------------------------------------------------------ + */ + + public function tableExists($tableName, $schemaName = null) + { + if ( empty($tableName) ) { + $msg = "Table name cannot be empty"; + $this->logAndThrowException($msg); + } + + $sql = "SELECT +table_name as NAME +FROM all_tables +WHERE owner = UPPER(:schema) +AND table_name = UPPER(:tablename)"; + + if ( null === $schemaName ) { + $schemaName = $this->getSchema(); + } + + $params = array( + ":schema" => $schemaName, + ":tablename" => $tableName + ); + + try { + $dbh = $this->getHandle(); + $result = $dbh->query($sql, $params); + if ( 0 == count($result) ) { + return false; + } + } catch (PDOException $e) { + $msg = "Error querying for table '$schema'.'$tableName':"; + $this->logAndThrowSqlException($sql, $e, $msg); + } + + return true; + + } // tableExists() + + /* ------------------------------------------------------------------------------------------ + * @see iRdbmsEndpoint::getTableColumnNames() + * + * Obtaining the columns for a table is straightforward, however views are another + * matter. ALL_VIEWS contains the view definition but there is no data dictionary view + * for views akin to all_tab_cols. For this we need to use "describe ". + * ------------------------------------------------------------------------------------------ + */ + + public function getTableColumnNames($tableName, $schemaName = null) + { + if ( empty($tableName) ) { + $msg = "Table name cannot be empty"; + $this->logAndThrowException($msg); + } + + $sql = "SELECT +column_name AS NAME, data_type AS type +FROM all_tab_cols +WHERE owner = UPPER(:schema) +AND table_name = UPPER(:tablename) +ORDER BY column_id ASC"; + + $params = array(":schema" => $this->getSchema(), + ":tablename" => $tableName); + + try { + $dbh = $this->getHandle(); + $result = $dbh->query($sql, $params); + if ( 0 == count($result) ) { + $msg = "No columns returned"; + $this->logAndThrowException($msg); + } + } catch (PDOException $e) { + $msg = "Error retrieving column names from '" . $this->getSchema() . ".'$tableName' "; + $this->logAndThrowSqlException($sql, $e, $msg); + } + + $columnNames = array(); + + foreach ( $result as $row ) { + $columnNames[] = $row['NAME']; + } + + return $columnNames; + + } // getTableColumnNames() +} // class Postgres diff --git a/classes/ETL/DataEndpoint/README_ORACLE.md b/classes/ETL/DataEndpoint/README_ORACLE.md new file mode 100644 index 0000000000..74d21763d4 --- /dev/null +++ b/classes/ETL/DataEndpoint/README_ORACLE.md @@ -0,0 +1,219 @@ +# Oracle Data Endpoint + +The Oracle data endpoint requires that Oracle client libraries are installed and the oci8 extension +is installed into PHP. The supported method for connecting to Oracle is via the Oracle Instant +Client Libraries and the PBP OCI8 extension (installed via PECL) + +## Oracle Instant Client Libraries and PHP OCI8 Module + +Download and install the [Oracle Instant Client]( http://www.oracle.com/technetwork/database/features/instant-client/index-097480.html). Both the basic and sdk files are needed to install the PECL module. **You will need an Oracle Account to download the files.** + +- instantclient-basic-linux.x64-12.1.0.2.0.zip +- instantclient-sdk-linux.x64-12.1.0.2.0.zip + +See: + +- http://php.net/manual/en/oci8.requirements.php +- http://php.net/manual/en/oci8.installation.php +- http://www.oracle.com/technetwork/database/features/instant-client/index-097480.html + +Installation instructions are below, however see the following notes: + +- **These instructions were tested on Ubuntu 14.04 using v12.1.0.2.0 of the Oracle libraries and + installing from PECL. You will need to make adjustments for your specific system and version.** +- **From the [oci8 page](https://pecl.php.net/package/oci8) use `pecl install oci8` for PHP 5.7 and + `pecl install oci8-2.0.12` to install for PHP 5.2 - PHP 5.6.** +- **When the pecl install asks for the Oracle home directory use `instantclient,/opt/oracle`.** + +Install the Oracle libraries: + +``` +sudo -s +cd /opt +unzip instantclient-basic-linux.x64-12.1.0.2.0.zip +unzip instantclient-sdk-linux.x64-12.1.0.2.0.zip +ln -s instantclient_12_1 oracle +chmod -R g+rX,o+rX instantclient_12_1 +cd instantclient_12_1 +ln -s libclntsh.so.12.1 libclntsh.so +ln -s libocci.so.12.1 libocci.so +pecl install oci8 +``` + +Set Up The PHP Module + +``` +sh -c 'echo extension=oci8.so > /etc/php5/cli/conf.d/oci8.ini' +sh -c 'echo extension=oci8.so > /etc/php5/apache2/conf.d/oci8.ini' +chmod 644 /etc/php5/cli/conf.d/oci8.ini +chmod 644 /etc/php5/apache2/conf.d/oci8.ini +``` + +Verify the installtion by running `php -i`: + +``` +oci8 + +OCI8 Support => enabled +OCI8 DTrace Support => disabled +OCI8 Version => 2.0.12 +Revision => $Id: 020312b6429ebb9d6272ac9bc28f6dce529434b6 $ +Oracle Run-time Client Library Version => 12.1.0.2.0 +Oracle Compile-time Instant Client Version => 12.1 +Directive => Local Value => Master Value + +oci8.connection_class => no value => no value +oci8.default_prefetch => 100 => 100 +oci8.events => Off => Off +oci8.max_persistent => -1 => -1 +oci8.old_oci_close_semantics => Off => Off +oci8.persistent_timeout => -1 => -1 +oci8.ping_interval => 60 => 60 +oci8.privileged_connect => Off => Off +oci8.statement_cache_size => 20 => 20 +``` + +## Configue and Test Oracle Connection + +A connection or Oracle can be made using the [Easy Connect Naming Method](https://docs.oracle.com/database/121/NETAG/naming.htm#NETAG008) or using a [Local Naming Method](https://docs.oracle.com/database/121/NETAG/naming.htm#NETAG081) using a `tnsnames.ora` file. + +**Note that when using the Oracle Instant Client, the `$ORACLE_HOME` environment variable is not set.** + +The examples below assume the setting below, modify these for your specific configuration: +- The database server is **db.mycompany.com** +- The port is the default 1521 +- The service name is **mydb** +- The database username is **scott** +- The database password is **tiger** +- The directory where the tnsnames.ora file resides is **~/oracle/tns** + +### Easy Connect Method + +No setup is required. Test the connection. + +``` + +``` + +### Local Naming Method + +Set up the network service names in `~/oracle/tns/tnsnames.ora` + +``` +datawarehouse = + (description = + (address = (protocol = tcp)(host = db.mycompany.com)(port = 1521)) + (connect_data = + (service_name = mydb))) +``` + +Set the `TNS_ADMIN` environment variable + +``` +export TNS_ADMIN=~/oracle/tns +``` + +Test the connection + +``` + +``` + +## Install PDOOCI + +Enabling Oracle support for PDO in PHP is painful. The obsolete +[PECL](https://pecl.php.net/package/pdo_oci) package can be installed or the PDO_OCI module can be +compiled from source (both of which are from 2005) using the Oracle SDK, which is not an Open Source +product. The OCI8 module is actively supported, but this does not support PDO. An alternative is the +[PDOOCI](https://github.com/taq/pdooci) project that wraps OCI functions to simulate a PDO object and +calls OCI under the hood. + +``` +composer require taq/pdooci +``` + +## Test + +PDOOCI + +``` +// Local Naming: Uses tnsnames.ora file in $TNS_ADMIN directory +$dsn = 'oci:dbname=DataWarehouse'; + +// Easy Connect Naming: //host[:port]/service +$dsn = 'oci:dbname=//oracle.domain.com:1521/datawarehouse' + +$dbh = @new \PDOOCI\PDO($dsn, $username, $db_password); +$sql = 'SELECT *FROM user_role_privs'; +$stmt = $dbh->prepare($sql); +$stmt->execute(); +$result = $stmt->fetchAll(\PDO::FETCH_ASSOC); +print_r($result); +``` + +Oracle DataEndpoint + +``` +$options = new DataEndpointOptions(); +$options->type = "oracle"; +$options->name = "My Infosource"; +$options->config = "oracle-test"; +$options->schema = "ENT"; +$myendpoint = DataEndpoint::factory($options); + +try { + $endpoint->verify(true, false); + print "Endpoint: " . $endpoint . PHP_EOL; + + $sql = 'SELECT *FROM user_role_privs'; + + $result = $endpoint->getHandle()->query($sql); + print_r($result); +} catch (Exception $e) { + exit($e->getMessage() . PHP_EOL); +} +``` From 999acea3061462865d01463ef01fc994fb1a0800 Mon Sep 17 00:00:00 2001 From: Steve Gallo Date: Fri, 20 Jan 2017 13:46:24 -0500 Subject: [PATCH 08/15] Style fixes as per phpcs --- classes/ETL/DbEntity/Query.php | 19 +++++----- classes/ETL/Ingestor/pdoIngestor.php | 53 +++++++++++++++++++--------- 2 files changed, 45 insertions(+), 27 deletions(-) diff --git a/classes/ETL/DbEntity/Query.php b/classes/ETL/DbEntity/Query.php index 7c5e5be3df..12166612f8 100644 --- a/classes/ETL/DbEntity/Query.php +++ b/classes/ETL/DbEntity/Query.php @@ -96,7 +96,7 @@ public function __construct($config, $systemQuoteChar = null, Log $logger = null if ( ! is_object($config) && is_string($config) ) { $config = $this->parseJsonFile($config, "Query Definition"); - } else if ( ! $config instanceof stdClass) { + } elseif ( ! $config instanceof stdClass) { $msg = __CLASS__ . ": Argument is not a filename or object"; $this->logAndThrowException($msg); } @@ -161,22 +161,22 @@ protected function initialize(stdClass $config, $force = false) if ( ! isset($config->records) ) { $errorMsg[] = "records property not found"; - } else if ( ! is_object($config->records) ) { + } elseif ( ! is_object($config->records) ) { $errorMsg[] = "records property must be an object"; } if ( ! isset($config->joins) ) { $errorMsg[] = "joins property not found"; - } else if ( ! is_array($config->joins) ) { + } elseif ( ! is_array($config->joins) ) { $errorMsg[] = "joins property must be an array"; - } else if ( 0 == count($config->joins) ) { + } elseif ( 0 == count($config->joins) ) { $errorMsg[] = "joins property must include as least one element"; } if ( isset($config->groupby) ) { if ( ! is_array($config->groupby) ) { $errorMsg[] = "groupby property must be an array"; - } else if ( 0 == count($config->groupby) ) { + } elseif ( 0 == count($config->groupby) ) { $errorMsg[] = "groupby property must include as least one element"; } } @@ -184,7 +184,7 @@ protected function initialize(stdClass $config, $force = false) if ( isset($config->orderby) ) { if ( ! is_array($config->orderby) ) { $errorMsg[] = "orderby property must be an array"; - } else if ( 0 == count($config->orderby) ) { + } elseif ( 0 == count($config->orderby) ) { $errorMsg[] = "orderby property must include as least one element"; } } @@ -286,7 +286,7 @@ public function addRecord($columnName, $formula) if ( null === $formula || "" === $formula ) { $msg = "Empty formula for column '$columnName' '$formula'"; $this->logAndThrowException($msg); - } else if ( array_key_exists($columnName, $this->records) ) { + } elseif ( array_key_exists($columnName, $this->records) ) { $msg = "Column '$columnName' already has a formula specified"; $this->logAndThrowException($msg); } @@ -639,7 +639,7 @@ public function addOverseerRestriction($restriction, $template) if ( ! is_string($restriction) || "" == $restriction ) { $msg = "Overseer restriction key must be a non-empty string"; $this->logAndThrowException($msg); - } else if ( ! is_string($template) || "" == $template ) { + } elseif ( ! is_string($template) || "" == $template ) { $msg = "Overseer restriction template must be a non-empty string"; $this->logAndThrowException($msg); } @@ -695,7 +695,7 @@ public function addOverseerRestrictionValue($restriction, $value) if ( ! is_string($restriction) || "" == $restriction ) { $msg = "Overseer restriction key must be a non-empty string"; $this->logAndThrowException($msg); - } else if ( ! is_string($value) || "" == $value ) { + } elseif ( ! is_string($value) || "" == $value ) { $msg = "Overseer restriction template must be a non-empty string"; $this->logAndThrowException($msg); } @@ -842,5 +842,4 @@ public function toJson($succinct = false, $includeSchema = false) { return json_encode($this->toJsonObj($succinct, $includeSchema)); } // toJson() - } // class Query diff --git a/classes/ETL/Ingestor/pdoIngestor.php b/classes/ETL/Ingestor/pdoIngestor.php index b5cb432e9b..188cb757f4 100644 --- a/classes/ETL/Ingestor/pdoIngestor.php +++ b/classes/ETL/Ingestor/pdoIngestor.php @@ -149,9 +149,11 @@ protected function initialize() if ( null === $this->etlSourceQuery && isset($this->parsedDefinitionFile->source_query) ) { $this->logger->debug("Create ETL source query object"); - $this->etlSourceQuery = new Query($this->parsedDefinitionFile->source_query, - $this->sourceEndpoint->getSystemQuoteChar(), - $this->logger); + $this->etlSourceQuery = new Query( + $this->parsedDefinitionFile->source_query, + $this->sourceEndpoint->getSystemQuoteChar(), + $this->logger + ); // If supported by the source query, set the date ranges here. These will be overriden // in the _execute() function with the current start/end dates but are needed here to @@ -223,7 +225,7 @@ protected function initialize() $this->logger->debug("Destination fields parsed from source query (table definition key '$etlTableKey'): " . implode(", ", $this->destinationFieldMappings[$etlTableKey])); $this->fullSourceToDestinationMapping = true; - } else if ( count($this->etlDestinationTableList) > 1 + } elseif ( count($this->etlDestinationTableList) > 1 && count($this->destinationFieldMappings) != count($this->etlDestinationTableList) ) { if ( 0 == count($this->destinationFieldMappings) ) { @@ -267,7 +269,13 @@ protected function initialize() $sourceQueryFields = $this->destinationFieldMappings[$etlTableKey]; $missing = array_diff($sourceQueryFields, $this->availableSourceQueryFields); if ( 0 != count($missing) ) { - $missing = array_map(function($k, $v) { return "$k = $v";}, array_keys($missing), $missing); + $missing = array_map( + function ($k, $v) { + return "$k = $v"; + }, + array_keys($missing), + $missing + ); $undefinedSourceQueryColumns[] = "Table '$etlTableKey' has undefined source query records for keys (" . implode(", ", $missing) . ")"; } @@ -306,7 +314,7 @@ protected function getDestinationFields() { if ( ! isset($this->parsedDefinitionFile->destination_record_map) ) { return null; - } else if ( ! is_object($this->parsedDefinitionFile->destination_record_map) ) { + } elseif ( ! is_object($this->parsedDefinitionFile->destination_record_map) ) { $msg = "destination_fields must be an object where keys match table definition keys"; $this->logAndThrowException($msg); } @@ -317,7 +325,7 @@ protected function getDestinationFields() if ( ! is_object($fieldMap) ) { $msg = "Destination field map for table '$etlTableKey' must be an object"; $this->logAndThrowException($msg); - } else if ( 0 == count(array_keys((array) $fieldMap)) ) { + } elseif ( 0 == count(array_keys((array) $fieldMap)) ) { $msg = "destination_record_map for '$etlTableKey' is empty"; $this->logger->warning($msg); } @@ -390,7 +398,7 @@ protected function getSourceData() // ER_LOCK_DEADLOCK: Deadlock found when trying to get lock; try restarting transaction if ( $srcStatement->errorCode() != "40001" ) { $this->logAndThrowSqlException($this->sourceQueryString, $e, "Error querying source"); - } else if ( $n_attempts > self::MAX_QUERY_ATTEMPTS ) { + } elseif ( $n_attempts > self::MAX_QUERY_ATTEMPTS ) { $msg = "Could not execute source query after " . self::MAX_QUERY_ATTEMPTS . " attempts. Exiting."; $this->logAndThrowException($msg); } @@ -399,7 +407,7 @@ protected function getSourceData() get_class($this) . ': Query was cancelled by server with error ' . $srcStatement->errorCode() . '. Retries left = ' . (self::MAX_QUERY_ATTEMPTS - $n_attempts) - ); + ); } } // while (FALSE == $query_success) @@ -420,7 +428,7 @@ protected function getSourceData() * ------------------------------------------------------------------------------------------ */ - protected function performPreExecuteTasks() { + protected function performPreExecuteTasks() { // ------------------------------------------------------------------------------------------ // Update the start/end dates for this query and get the source query string. It is important @@ -602,7 +610,12 @@ private function singleDatabaseIngest() $sql = "REPLACE INTO $qualifiedDestTableName (" . implode(',', $destColumnList) . ")\n" . $this->sourceQueryString; } else { $destColumns = implode(',', $destColumnList); - $updateColumnList = array_map(function($s) { return "$s=VALUES($s)"; }, $destColumnList); + $updateColumnList = array_map( + function ($s) { + return "$s=VALUES($s)"; + }, + $destColumnList + ); $updateColumns = implode(',', $updateColumnList); $sql = "INSERT INTO $qualifiedDestTableName ($destColumns) " . $this->sourceQueryString . "\nON DUPLICATE KEY UPDATE $updateColumns"; @@ -649,8 +662,10 @@ private function multiDatabaseIngest() continue; } - $infileName = tempnam(sys_get_temp_dir(), - sprintf('%s.data.ts_%s.%s', $etlTable->getFullName(false), time(), rand())); + $infileName = tempnam( + sys_get_temp_dir(), + sprintf('%s.data.ts_%s.%s', $etlTable->getFullName(false), time(), rand()) + ); $this->logger->debug("Using temporary file '$infileName' for destination table key '$etlTableKey'"); @@ -676,7 +691,12 @@ private function multiDatabaseIngest() . $this->destinationEndpoint->quoteSystemIdentifier("tmp_" . $etlTable->getName() . "_" . time()); $destColumns = implode(',', $destColumnList); - $updateColumnList = array_map(function($s) { return "$s=VALUES($s)"; }, $destColumnList); + $updateColumnList = array_map( + function ($s) { + return "$s=VALUES($s)"; + }, + $destColumnList + ); $updateColumns = implode(',', $updateColumnList); $loadStatement = "CREATE TABLE $tmpTable LIKE $qualifiedDestTableName; " @@ -784,9 +804,9 @@ private function multiDatabaseIngest() foreach ( $srcRow as $key => &$value ) { if ( 'order_id' == $key) { $value = $totalRowsProcessed; - } else if ( NULL === $value ) { + } elseif ( null === $value ) { $value = '\N'; - } else if ( empty($value) ) { + } elseif ( empty($value) ) { $value = $stringEnc . '' . $stringEnc; } } @@ -940,5 +960,4 @@ protected function allowSingleDatabaseOptimization() return true; } // allowSingleDatabaseOptimization() - } // class pdoIngestor From 9c49f026e2636a1a88516a2e2688516aa8b8c9ac Mon Sep 17 00:00:00 2001 From: Steve Gallo Date: Fri, 20 Jan 2017 16:38:48 -0500 Subject: [PATCH 09/15] Addressed comments from @jsperhac --- classes/ETL/DataEndpoint/Mysql.php | 24 +--------- classes/ETL/DataEndpoint/Oracle.php | 24 +--------- classes/ETL/DataEndpoint/Postgres.php | 22 +--------- classes/ETL/DataEndpoint/aRdbmsEndpoint.php | 44 ++++++++++++++++++- .../etl/data-endpoint-oracle.md | 0 5 files changed, 46 insertions(+), 68 deletions(-) rename classes/ETL/DataEndpoint/README_ORACLE.md => docs/etl/data-endpoint-oracle.md (100%) diff --git a/classes/ETL/DataEndpoint/Mysql.php b/classes/ETL/DataEndpoint/Mysql.php index 1b7f366be9..db741a90ba 100644 --- a/classes/ETL/DataEndpoint/Mysql.php +++ b/classes/ETL/DataEndpoint/Mysql.php @@ -77,32 +77,12 @@ public function isSameServer(iDataEndpoint $cmp) public function schemaExists($schemaName = null) { - if ( null === $schemaName ) { - $schemaName = $this->getSchema(); - } elseif ( empty($schemaName) ) { - $msg = "Schema name cannot be empty"; - $this->logAndThrowException($msg); - } - $sql = "SELECT schema_name as name, catalog_name as catalog FROM information_schema.schemata WHERE schema_name = :schema"; - $params = array(":schema" => $schemaName); - - try { - $dbh = $this->getHandle(); - $result = $dbh->query($sql, $params); - if ( 0 == count($result) ) { - return false; - } - } catch (\PdoException $e) { - $msg = "Error querying for schema '$schemaName'"; - $this->logAndThrowSqlException($sql, $e, $msg); - } - - return true; + return $this->executeSchemaExistsQuery($sql, $schemaName); } // schemaExists() @@ -127,8 +107,6 @@ public function createSchema($schemaName = null) // Don't use bind parameters because we don't want to quote the schema $sql = "CREATE DATABASE IF NOT EXISTS $schemaName"; - // $params = array(":schema" => $this->getSchema()); - try { $dbh = $this->getHandle(); $result = $dbh->execute($sql); diff --git a/classes/ETL/DataEndpoint/Oracle.php b/classes/ETL/DataEndpoint/Oracle.php index afd5fa2124..0c5cd1ea8c 100644 --- a/classes/ETL/DataEndpoint/Oracle.php +++ b/classes/ETL/DataEndpoint/Oracle.php @@ -41,34 +41,12 @@ public function isSameServer(iDataEndpoint $cmp) public function schemaExists($schemaName = null) { - if ( null === $schemaName ) { - $schemaName = $this->getSchema(); - } elseif ( empty($schemaName) ) { - $msg = "Schema name cannot be empty"; - $this->logAndThrowException($msg); - } - - // See http://www.postgresql.org/docs/current/static/catalogs.html - $sql = "SELECT username AS name FROM all_users WHERE username = UPPER(:schema)"; - $params = array(":schema" => $this->getSchema()); - - try { - $dbh = $this->getHandle(); - $result = $dbh->query($sql, $params); - if ( 0 == count($result) ) { - return false; - } - } catch (\PdoException $e) { - $msg = "Error querying for schema '" . $this->getSchema() . "'"; - $this->logAndThrowSqlException($sql, $e, $msg); - } - - return true; + return $this->executeSchemaExistsQuery($sql, $schemaName); } // schemaExists() diff --git a/classes/ETL/DataEndpoint/Postgres.php b/classes/ETL/DataEndpoint/Postgres.php index fc5172f25e..02ef08cdb3 100644 --- a/classes/ETL/DataEndpoint/Postgres.php +++ b/classes/ETL/DataEndpoint/Postgres.php @@ -41,13 +41,6 @@ public function isSameServer(iDataEndpoint $cmp) public function schemaExists($schemaName = null) { - if ( null === $schemaName ) { - $schemaName = $this->getSchema(); - } elseif ( empty($schemaName) ) { - $msg = "Schema name cannot be empty"; - $this->logAndThrowException($msg); - } - // See http://www.postgresql.org/docs/current/static/catalogs.html $sql = "SELECT @@ -55,20 +48,7 @@ public function schemaExists($schemaName = null) FROM pg_catalog.pg_namespace WHERE nspname = :schema"; - $params = array(":schema" => $schemaName); - - try { - $dbh = $this->getHandle(); - $result = $dbh->query($sql, $params); - if ( 0 == count($result) ) { - return false; - } - } catch (\PdoException $e) { - $msg = "Error querying for schema '$schemaName'"; - $this->logAndThrowSqlException($sql, $e, $msg); - } - - return true; + return $this->executeSchemaExistsQuery($sql, $schemaName); } // schemaExists() diff --git a/classes/ETL/DataEndpoint/aRdbmsEndpoint.php b/classes/ETL/DataEndpoint/aRdbmsEndpoint.php index 28723dc06a..dc2218f0e4 100644 --- a/classes/ETL/DataEndpoint/aRdbmsEndpoint.php +++ b/classes/ETL/DataEndpoint/aRdbmsEndpoint.php @@ -133,7 +133,7 @@ public function connect() $this->handle = DB::factory($this->config); } catch (Exception $e) { $msg = "Error connecting to data endpoint '" . $this->name . "'. " . $e->getMessage(); - $this->logAndThrowException($msg); + $this->logAndThrowException($msg); } return $this->handle; @@ -270,6 +270,48 @@ public function getTableColumnNames($tableName, $schemaName = null) } // getTableColumnNames() + /* ------------------------------------------------------------------------------------------ + * Helper function used by specific data endpoint drivers to query the underlying + * database to check if a schema exists. + * + * @param $sql The SQL statement used to query for the schema + * @param $schemaName The name of the schema, or NULL to use the default schema for + * this endpoint + * @param $sqlParameters An array of additional parameters for the SQL + * statement. Parameters here will override local parameters with the same key + * + * @return TRUE if the schema exists, FALSE if it does not + * @throw Exception If an empty and non-NULL schema was provided + * @throw Exception If tehre was an errir querying the database + * ------------------------------------------------------------------------------------------ + */ + + protected function executeSchemaExistsQuery($sql, $schemaName = null, array $sqlParameters = array()) + { + if ( null === $schemaName ) { + $schemaName = $this->getSchema(); + } elseif ( empty($schemaName) ) { + $msg = "Schema name cannot be empty"; + $this->logAndThrowException($msg); + } + + $localSqlParameters = array(":schema" => $schemaName); + $localSqlParameters = array_merge($localSqlParameters, $sqlParameters); + + try { + $dbh = $this->getHandle(); + $result = $dbh->query($sql, $localSqlParameters); + if ( 0 == count($result) ) { + return false; + } + } catch (\PdoException $e) { + $msg = "Error querying for schema '$schemaName'"; + $this->logAndThrowSqlException($sql, $e, $msg); + } + + return true; + } // executeSchemaExistsQuery() + /* ------------------------------------------------------------------------------------------ * @see aDataEndpoint::__toString() * ------------------------------------------------------------------------------------------ diff --git a/classes/ETL/DataEndpoint/README_ORACLE.md b/docs/etl/data-endpoint-oracle.md similarity index 100% rename from classes/ETL/DataEndpoint/README_ORACLE.md rename to docs/etl/data-endpoint-oracle.md From 0b084898e6d975bbc25d83ef7ded1d2409db9f0e Mon Sep 17 00:00:00 2001 From: Steve Gallo Date: Fri, 20 Jan 2017 19:55:54 -0500 Subject: [PATCH 10/15] Address style and unit test errors --- classes/CCR/DB.php | 2 +- classes/CCR/DB/NullDB.php | 14 +- tools/etl/etl_overseer.php | 533 ++++++++++++++++++++----------------- 3 files changed, 296 insertions(+), 253 deletions(-) diff --git a/classes/CCR/DB.php b/classes/CCR/DB.php index b5f171ca28..2b4cc268ce 100644 --- a/classes/CCR/DB.php +++ b/classes/CCR/DB.php @@ -66,7 +66,7 @@ public static function factory($sectionName, $autoConnect = true) $iniSection = xd_utilities\getConfigurationSection($sectionName, 'db_engine'); } catch (Exception $e) { $msg = "Unable to get database configuration options: " . $e->getMessage(); - throw new Exception ($msg); + throw new Exception($msg); } // Not all engines are required to specify all configuration options (e.g., Oracle) so diff --git a/classes/CCR/DB/NullDB.php b/classes/CCR/DB/NullDB.php index 32144f3cc6..266a8cb8f8 100644 --- a/classes/CCR/DB/NullDB.php +++ b/classes/CCR/DB/NullDB.php @@ -14,11 +14,19 @@ class NullDB implements iDatabase { + public function __construct($db_engine, $db_host, $db_port, $db_name, $db_username, $db_password) + { + } + + public function _destruct() + { + } + public function connect() { } - public function destroy() + public function disconnect() { } @@ -27,6 +35,10 @@ public function insert($statement, $params = array()) return 0; } + public function handle() + { + } + public function query( $query, array $params = array(), diff --git a/tools/etl/etl_overseer.php b/tools/etl/etl_overseer.php index ab57a91b75..ede3e017d6 100755 --- a/tools/etl/etl_overseer.php +++ b/tools/etl/etl_overseer.php @@ -1,11 +1,11 @@ #!/usr/bin/env php - */ +/** + * Perform ETL on federated resources. This is different than the traditional ETL process in that + * it uses a new mechanism for passing options to the ingesters and is (hopefully) more flexible. + * + * @author Steve Gallo + */ require __DIR__ . '/../../configuration/linker.php'; restore_exception_handler(); @@ -72,7 +72,7 @@ // ETL start date 'start-date' => null, 'verbosity' => Log::NOTICE - ); +); $showList = false; @@ -99,179 +99,195 @@ 'v:' => 'verbosity:', 'x:' => 'exclude-resource-codes:', 'y:' => 'last-modified-end-date:' - ); +); $args = getopt(implode('', array_keys($options)), $options); foreach ($args as $arg => $value) { switch ($arg) { - case 'a': - case 'action': - // Merge array because long and short options are grouped separately - $scriptOptions['actions'] = array_merge($scriptOptions['actions'], - ( is_array($value) ? $value : array($value) )); - break; - - case 'b': - case 'base-dir': - if ( ! is_dir($value) ) usage_and_exit("Base directory does not exist: '$value'"); - $scriptOptions['base-dir'] = $value; - break; - - case 'k': - case 'chunk-size-days': - switch ( $value ) { - case 'none': - $scriptOptions['chunk-size-days'] = null; + case 'a': + case 'action': + // Merge array because long and short options are grouped separately + $scriptOptions['actions'] = array_merge( + $scriptOptions['actions'], + ( is_array($value) ? $value : array($value) ) + ); break; - case 'day': - $scriptOptions['chunk-size-days'] = 1; + + case 'b': + case 'base-dir': + if ( ! is_dir($value) ) { + usage_and_exit("Base directory does not exist: '$value'"); + } + $scriptOptions['base-dir'] = $value; break; - case 'week': - $scriptOptions['chunk-size-days'] = 7; + + case 'k': + case 'chunk-size-days': + switch ( $value ) { + case 'none': + $scriptOptions['chunk-size-days'] = null; + break; + case 'day': + $scriptOptions['chunk-size-days'] = 1; + break; + case 'week': + $scriptOptions['chunk-size-days'] = 7; + break; + case 'month': + $scriptOptions['chunk-size-days'] = 30; + break; + case 'year': + $scriptOptions['chunk-size-days'] = 365; + break; + default: + usage_and_exit("Invalid chunk size: $value"); + break; + } break; - case 'month': - $scriptOptions['chunk-size-days'] = 30; + + case 'c': + case 'config-file': + $scriptOptions['config-file'] = $value; break; - case 'year': - $scriptOptions['chunk-size-days'] = 365; + + case 't': + case 'dryrun': + $scriptOptions['dryrun'] = true; break; - default: - usage_and_exit("Invalid chunk size: $value"); + + case 'e': + case 'end-date': + if ( false === strtotime($value) ) { + usage_and_exit("Could not parse end date: '$value'"); + } + $scriptOptions['end-date'] = $value; break; - } - break; - - case 'c': - case 'config-file': - $scriptOptions['config-file'] = $value; - break; - - case 't': - case 'dryrun': - $scriptOptions['dryrun'] = true; - break; - - case 'e': - case 'end-date': - if ( false === strtotime($value) ) usage_and_exit("Could not parse end date: '$value'"); - $scriptOptions['end-date'] = $value; - break; - - case 'f': - case 'force': - $scriptOptions['force'] = true; - break; - - case 'g': - case 'group': - $scriptOptions['groups'] = ( is_array($value) ? $value : array($value) ); - break; - - case 'l': - case 'list': - $showList = true; - $value = ( is_array($value) ? $value : array($value) ); - foreach ( $value as $type ) { - $key = "list-" . $type; - $scriptOptions[$key] = true; - } // foreach ( $value as $type ) - break; - - case 'm': - case 'last-modified-start-date': - if ( false === strtotime($value) ) usage_and_exit("Could not parse last modified start date: '$value'"); - $scriptOptions['last-modified-start-date'] = $value; - break; - - case 'n': - case 'number-of-days': - $scriptOptions['number-of-days'] = filter_var($value, FILTER_VALIDATE_INT); - if ($scriptOptions['number-of-days'] < 1) { - usage_and_exit("$arg must be an integer greater than 0"); - } - break; - - case 'o': - case 'option': - $value = ( is_array($value) ? $value : array($value) ); - foreach ( $value as $option ) { - $parts = explode("=", $option); - if ( 2 != count($parts) ) { - usage_and_exit("Options must be of the form option=value: '$option'"); + + case 'f': + case 'force': + $scriptOptions['force'] = true; + break; + + case 'g': + case 'group': + $scriptOptions['groups'] = ( is_array($value) ? $value : array($value) ); + break; + + case 'l': + case 'list': + $showList = true; + $value = ( is_array($value) ? $value : array($value) ); + foreach ( $value as $type ) { + $key = "list-" . $type; + $scriptOptions[$key] = true; + } // foreach ( $value as $type ) + break; + + case 'm': + case 'last-modified-start-date': + if ( false === strtotime($value) ) { + usage_and_exit("Could not parse last modified start date: '$value'"); } - $scriptOptions['option-overrides'][trim($parts[0])] = trim($parts[1]); - } - break; - - case 'r': - case 'only-resource-codes': - // Merge array because long and short options are grouped separately - $scriptOptions['include-only-resource-codes'] = - array_merge($scriptOptions['include-only-resource-codes'], - ( is_array($value) ? $value : array($value) ) ); - break; - - case 'p': - case 'process-section': - // Merge array because long and short options are grouped separately - $scriptOptions['process-sections'] = array_merge($scriptOptions['process-sections'], - ( is_array($value) ? $value : array($value) )); - break; - - case 's': - case 'start-date': - if ( false === strtotime($value) ) usage_and_exit("Could not parse start date: '$value'"); - $scriptOptions['start-date'] = $value; - break; - - case 'v': - case 'verbosity': - switch ( $value ) { - case 'debug': - $scriptOptions['verbosity'] = Log::DEBUG; + $scriptOptions['last-modified-start-date'] = $value; break; - case 'info': - $scriptOptions['verbosity'] = Log::INFO; + + case 'n': + case 'number-of-days': + $scriptOptions['number-of-days'] = filter_var($value, FILTER_VALIDATE_INT); + if ($scriptOptions['number-of-days'] < 1) { + usage_and_exit("$arg must be an integer greater than 0"); + } break; - case 'notice': - $scriptOptions['verbosity'] = Log::NOTICE; + + case 'o': + case 'option': + $value = ( is_array($value) ? $value : array($value) ); + foreach ( $value as $option ) { + $parts = explode("=", $option); + if ( 2 != count($parts) ) { + usage_and_exit("Options must be of the form option=value: '$option'"); + } + $scriptOptions['option-overrides'][trim($parts[0])] = trim($parts[1]); + } break; - case 'warning': - $scriptOptions['verbosity'] = Log::WARNING; + + case 'r': + case 'only-resource-codes': + // Merge array because long and short options are grouped separately + $scriptOptions['include-only-resource-codes'] = array_merge( + $scriptOptions['include-only-resource-codes'], + ( is_array($value) ? $value : array($value) ) + ); break; - case 'quiet': - $scriptOptions['verbosity'] = Log::EMERG; + + case 'p': + case 'process-section': + // Merge array because long and short options are grouped separately + $scriptOptions['process-sections'] = array_merge( + $scriptOptions['process-sections'], + ( is_array($value) ? $value : array($value) ) + ); break; + + case 's': + case 'start-date': + if ( false === strtotime($value) ) { + usage_and_exit("Could not parse start date: '$value'"); + } + $scriptOptions['start-date'] = $value; + break; + + case 'v': + case 'verbosity': + switch ( $value ) { + case 'debug': + $scriptOptions['verbosity'] = Log::DEBUG; + break; + case 'info': + $scriptOptions['verbosity'] = Log::INFO; + break; + case 'notice': + $scriptOptions['verbosity'] = Log::NOTICE; + break; + case 'warning': + $scriptOptions['verbosity'] = Log::WARNING; + break; + case 'quiet': + $scriptOptions['verbosity'] = Log::EMERG; + break; + default: + usage_and_exit("Invalid verbosity level: $value"); + break; + } // switch ( $value ) + break; + + case 'x': + case 'exclude-resource-codes': + // Merge array because long and short options are grouped separately + $scriptOptions['exclude-resource-codes'] = array_merge( + $scriptOptions['exclude-resource-codes'], + ( is_array($value) ? $value : array($value) ) + ); + break; + + case 'y': + case 'last-modified-end-date': + if ( false === strtotime($value) ) { + usage_and_exit("Could not parse last modified end date: '$value'"); + } + $scriptOptions['last-modified-end-date'] = $value; + break; + + case 'h': + case 'help': + usage_and_exit(); + break; + default: - usage_and_exit("Invalid verbosity level: $value"); + usage_and_exit("Invalid option: $arg"); break; - } // switch ( $value ) - break; - - case 'x': - case 'exclude-resource-codes': - // Merge array because long and short options are grouped separately - $scriptOptions['exclude-resource-codes'] = - array_merge($scriptOptions['exclude-resource-codes'], - ( is_array($value) ? $value : array($value) ) ); - break; - - case 'y': - case 'last-modified-end-date': - if ( false === strtotime($value) ) usage_and_exit("Could not parse last modified end date: '$value'"); - $scriptOptions['last-modified-end-date'] = $value; - break; - - case 'h': - case 'help': - usage_and_exit(); - break; - - default: - usage_and_exit("Invalid option: $arg"); - break; } } // foreach ($args as $arg => $value) @@ -287,7 +303,7 @@ if ( 0 === strpos($arg, "--") ) { $opt = substr($arg, 2); - } else if ( 0 === strpos($arg, "-") ) { + } elseif ( 0 === strpos($arg, "-") ) { $opt = substr($arg, 1); } if ( null !== $opt && ! in_array($opt, $parsedArgs) ) { @@ -307,9 +323,11 @@ $conf = array( 'emailSubject' => gethostname() . ': XDMOD: Data Warehouse: Federated ETL Log', 'mail' => false - ); +); -if ( null !== $scriptOptions['verbosity'] ) $conf['consoleLogLevel'] = $scriptOptions['verbosity']; +if ( null !== $scriptOptions['verbosity'] ) { + $conf['consoleLogLevel'] = $scriptOptions['verbosity']; +} $logger = Log::factory('DWI', $conf); @@ -318,7 +336,7 @@ if ( null === $scriptOptions['config-file'] ) { usage_and_exit("Config file required"); -} else if ( null !== $scriptOptions['config-file'] && ! is_file($scriptOptions['config-file']) ) { +} elseif ( null !== $scriptOptions['config-file'] && ! is_file($scriptOptions['config-file']) ) { usage_and_exit("Config file not found: '" . $scriptOptions['config-file'] . "'"); } @@ -330,18 +348,20 @@ if ( ! $showList) { $logger->notice(array( - 'message' => 'dw_extract_transform_load start', - 'process_start_time' => date('Y-m-d H:i:s'), - )); + 'message' => 'dw_extract_transform_load start', + 'process_start_time' => date('Y-m-d H:i:s'), + )); } // ------------------------------------------------------------------------------------------ // Parse the ETL configuration. We will need it for listing available ingestors, aggregators, etc. try { - $etlConfig = new EtlConfiguration($scriptOptions['config-file'], - $scriptOptions['base-dir'], - $scriptOptions['option-overrides']); + $etlConfig = new EtlConfiguration( + $scriptOptions['config-file'], + $scriptOptions['base-dir'], + $scriptOptions['option-overrides'] + ); $etlConfig->setLogger($logger); $etlConfig->initialize(); } catch ( Exception $e ) { @@ -362,11 +382,13 @@ $missing = array(); foreach ( $scriptOptions['process-sections'] as $sectionName ) { - if ( ! $etlConfig->sectionExists($sectionName) ) $missing[] = $sectionName; + if ( ! $etlConfig->sectionExists($sectionName) ) { + $missing[] = $sectionName; + } } if ( count($missing) > 0 ) { - fwrite(STDERR, "Unknown sections: " . implode(", " , $missing) . "\n"); + fwrite(STDERR, "Unknown sections: " . implode(", ", $missing) . "\n"); exit(); } } // if ( count($scriptOptions['process-sections'] > 0) ) @@ -381,98 +403,107 @@ } $utilitySchema = $utilityEndpoint->getSchema(); -$listOptions = array_filter(array_keys($scriptOptions), function($key) { +$listOptions = array_filter( + array_keys($scriptOptions), + function ($key) { return "list-" == substr($key, 0, 5); - }); + } +); if ( $showList ) { foreach ( $listOptions as $opt ) { - if ( ! $scriptOptions[$opt] ) continue; + if ( ! $scriptOptions[$opt] ) { + continue; + } switch ( $opt ) { - case 'list-resources': - $sql = "SELECT code, start_date, end_date from {$utilitySchema}.resourcefact WHERE resourcetype_id NOT IN (0,4) ORDER BY CODE ASC"; - try { - $result = $utilityEndpoint->getHandle()->query($sql); - } catch (Exception $e) { - exit($e->getMessage() . "\n". $e->getTraceAsString() . "\n"); - } - $headings = array("Resource Code","Start Date","End Date"); - print implode(LIST_SEPARATOR, $headings) . "\n"; - - foreach ( $result as $row ) { - $fields = array($row['code'], $row['start_date'], $row['end_date']); - print implode(LIST_SEPARATOR, $fields) . "\n"; - } - break; - - case 'list-sections': - $sectionNames = $etlConfig->getSectionNames(); - sort($sectionNames); - print "Section\n"; - foreach ( $sectionNames as $name ) print "$name\n"; - break; + case 'list-resources': + $sql = "SELECT code, start_date, end_date from {$utilitySchema}.resourcefact WHERE resourcetype_id NOT IN (0,4) ORDER BY CODE ASC"; + try { + $result = $utilityEndpoint->getHandle()->query($sql); + } catch (Exception $e) { + exit($e->getMessage() . "\n". $e->getTraceAsString() . "\n"); + } + $headings = array("Resource Code","Start Date","End Date"); + print implode(LIST_SEPARATOR, $headings) . "\n"; - case 'list-actions': - $sectionNames = $etlConfig->getSectionNames(); - sort($sectionNames); - $headings = array("Section","Action","Status","Description"); - print implode(LIST_SEPARATOR, $headings) . "\n"; - - foreach ( $sectionNames as $sectionName ) { - $actions = $etlConfig->getConfiguredActionNames($sectionName); - foreach ( $actions as $actionName ) { - $options = $etlConfig->getActionOptions($actionName, $sectionName); - $fields = array($sectionName, $actionName, ( $options->enabled ? "enabled" : "disabled"), $options->description); + foreach ( $result as $row ) { + $fields = array($row['code'], $row['start_date'], $row['end_date']); print implode(LIST_SEPARATOR, $fields) . "\n"; } - } - break; - - case 'list-endpoints': - $endpoints = $etlConfig->getDataEndpoints(); + break; + + case 'list-sections': + $sectionNames = $etlConfig->getSectionNames(); + sort($sectionNames); + print "Section\n"; + foreach ( $sectionNames as $name ) { + print "$name\n"; + } + break; - $endpointSummary = array(); - foreach ( $endpoints as $endpoint ) { - $endpointSummary[0][] = $endpoint->getType(); - $endpointSummary[1][] = $endpoint->getName(); - $endpointSummary[2][] = $endpoint->getKey(); - $endpointSummary[3][] = (string) $endpoint; + case 'list-actions': + $sectionNames = $etlConfig->getSectionNames(); + sort($sectionNames); + $headings = array("Section","Action","Status","Description"); + print implode(LIST_SEPARATOR, $headings) . "\n"; - } - array_multisort($endpointSummary[0], $endpointSummary[1], $endpointSummary[2], $endpointSummary[3]); + foreach ( $sectionNames as $sectionName ) { + $actions = $etlConfig->getConfiguredActionNames($sectionName); + foreach ( $actions as $actionName ) { + $options = $etlConfig->getActionOptions($actionName, $sectionName); + $fields = array($sectionName, $actionName, ( $options->enabled ? "enabled" : "disabled"), $options->description); + print implode(LIST_SEPARATOR, $fields) . "\n"; + } + } + break; - $headings = array("Type","Name","Key","Description"); - print implode(LIST_SEPARATOR, $headings) . "\n"; + case 'list-endpoints': + $endpoints = $etlConfig->getDataEndpoints(); - for ( $i=0; $i < count($endpointSummary[0]); $i++ ) { - $a = array($endpointSummary[0][$i], - $endpointSummary[1][$i], - $endpointSummary[2][$i], - $endpointSummary[3][$i]); - print implode(LIST_SEPARATOR, $a) . "\n";; - } - break; + $endpointSummary = array(); + foreach ( $endpoints as $endpoint ) { + $endpointSummary[0][] = $endpoint->getType(); + $endpointSummary[1][] = $endpoint->getName(); + $endpointSummary[2][] = $endpoint->getKey(); + $endpointSummary[3][] = (string) $endpoint; - case 'list-groups': - // Groups are not supported yet. - break; + } + array_multisort($endpointSummary[0], $endpointSummary[1], $endpointSummary[2], $endpointSummary[3]); - default: - // Remove the "list-" prefix to get the section name - $sectionName = substr($opt, 5); - if ( false !== ($actions = $etlConfig->getConfiguredActionNames($sectionName)) ) { - $headings = array("Action","Status","Description"); + $headings = array("Type","Name","Key","Description"); print implode(LIST_SEPARATOR, $headings) . "\n"; - foreach ( $actions as $actionName ) { - $options = $etlConfig->getActionOptions($actionName, $sectionName); - $fields = array($actionName, $sectionName, ( $options->enabled ? "enabled" : "disabled"), $options->description); - print implode(LIST_SEPARATOR, $fields) . "\n"; + + for ( $i=0; $i < count($endpointSummary[0]); $i++ ) { + $a = array( + $endpointSummary[0][$i], + $endpointSummary[1][$i], + $endpointSummary[2][$i], + $endpointSummary[3][$i] + ); + print implode(LIST_SEPARATOR, $a) . "\n";; } - } else { - print "Unknown section name: '$sectionName'\n"; - } - break; + break; + + case 'list-groups': + // Groups are not supported yet. + break; + + default: + // Remove the "list-" prefix to get the section name + $sectionName = substr($opt, 5); + if ( false !== ($actions = $etlConfig->getConfiguredActionNames($sectionName)) ) { + $headings = array("Action","Status","Description"); + print implode(LIST_SEPARATOR, $headings) . "\n"; + foreach ( $actions as $actionName ) { + $options = $etlConfig->getActionOptions($actionName, $sectionName); + $fields = array($actionName, $sectionName, ( $options->enabled ? "enabled" : "disabled"), $options->description); + print implode(LIST_SEPARATOR, $fields) . "\n"; + } + } else { + print "Unknown section name: '$sectionName'\n"; + } + break; } } exit(); @@ -487,7 +518,7 @@ // If there is no end date, assume today $scriptOptions['end-date'] = date("Y-m-d 23:59:59"); $logger->info("No end date set, assuming " . $scriptOptions['end-date']); - } else if ( null === $scriptOptions['start-date'] ) { + } elseif ( null === $scriptOptions['start-date'] ) { // If no start date assume epoch (1970-01-01 00:00:00) in the current timezone $scriptOptions['start-date'] = "1970-01-01 00:00:00"; $logger->info("No start date set, assuming " . $scriptOptions['start-date']); @@ -596,7 +627,7 @@ function usage_and_exit($msg = null) fwrite( STDERR, -<<<"EOMSG" + <<<"EOMSG" Usage: {$argv[0]} -h, --help @@ -658,8 +689,8 @@ function usage_and_exit($msg = null) NOTE: Date and time options support "+1 day", "now", "now - 1 day", etc. notation. -EOMSG - ); + EOMSG; + ); exit(1); } From d603ce2b21d5fa5998633309df13762054eddc53 Mon Sep 17 00:00:00 2001 From: Steve Gallo Date: Sat, 21 Jan 2017 09:49:42 -0500 Subject: [PATCH 11/15] TravisCI build fixes --- classes/CCR/DB.php | 13 +++++++++---- classes/CCR/DB/NullDB.php | 8 ++++++-- tools/etl/etl_overseer.php | 4 ++-- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/classes/CCR/DB.php b/classes/CCR/DB.php index 2b4cc268ce..6c1e1d9f36 100644 --- a/classes/CCR/DB.php +++ b/classes/CCR/DB.php @@ -30,13 +30,17 @@ class DB // Ensure that this class is a singleton - private function __construct() {} + private function __construct() + { + } // ================================================================================ // Cleanup // ================================================================================ - public function __destruct() {} + public function __destruct() + { + } // ================================================================================ // Create an instance of the database singleton. A single argument is @@ -103,10 +107,11 @@ public static function factory($sectionName, $autoConnect = true) } self::$instancePool[$sectionName] = $db; - if ($autoConnect) self::$instancePool[$sectionName]->connect(); + if ($autoConnect) { + self::$instancePool[$sectionName]->connect(); + } return self::$instancePool[$sectionName]; } // factory() - } // class DB diff --git a/classes/CCR/DB/NullDB.php b/classes/CCR/DB/NullDB.php index 266a8cb8f8..7d31062045 100644 --- a/classes/CCR/DB/NullDB.php +++ b/classes/CCR/DB/NullDB.php @@ -18,7 +18,7 @@ public function __construct($db_engine, $db_host, $db_port, $db_name, $db_userna { } - public function _destruct() + public function __destruct() { } @@ -52,9 +52,13 @@ public function execute($query, array $params = array()) return 0; } + public function getRowCount($schema, $table) + { + } + public function prepare($query) { - return FALSE; + return false; } public function beginTransaction() diff --git a/tools/etl/etl_overseer.php b/tools/etl/etl_overseer.php index ede3e017d6..e8382370d7 100755 --- a/tools/etl/etl_overseer.php +++ b/tools/etl/etl_overseer.php @@ -481,7 +481,7 @@ function ($key) { $endpointSummary[2][$i], $endpointSummary[3][$i] ); - print implode(LIST_SEPARATOR, $a) . "\n";; + print implode(LIST_SEPARATOR, $a) . "\n"; } break; @@ -689,7 +689,7 @@ function usage_and_exit($msg = null) NOTE: Date and time options support "+1 day", "now", "now - 1 day", etc. notation. - EOMSG; +EOMSG ); exit(1); From c0e48e3b36760429dd085e082a8e2d7ec2a18386 Mon Sep 17 00:00:00 2001 From: Steve Gallo Date: Sat, 21 Jan 2017 09:58:53 -0500 Subject: [PATCH 12/15] More style fixes --- tools/etl/etl_overseer.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/etl/etl_overseer.php b/tools/etl/etl_overseer.php index f918d96d85..4b0a5fa976 100755 --- a/tools/etl/etl_overseer.php +++ b/tools/etl/etl_overseer.php @@ -92,7 +92,7 @@ if ( array_key_exists($configKey, $scriptOptions) ) { $scriptOptions[$configKey] = $configValue; - } else if ( array_key_exists($dashKey, $scriptOptions) ) { + } elseif ( array_key_exists($dashKey, $scriptOptions) ) { $scriptOptions[$dashKey] = $configValue; } } // foreach ( $etlConfigOptions as $configkey => $configValue ) From 57a3c08793f3cfcd97d8e198c848b058cdeae057 Mon Sep 17 00:00:00 2001 From: Steve Gallo Date: Sat, 21 Jan 2017 12:40:08 -0500 Subject: [PATCH 13/15] Remove __construct() from iDatabase, clean up code style --- classes/CCR/DB/MySQLDB.php | 12 +++++------- classes/CCR/DB/NullDB.php | 2 +- classes/CCR/DB/PostgresDB.php | 34 ++++++++++++++++------------------ classes/CCR/DB/iDatabase.php | 18 +----------------- 4 files changed, 23 insertions(+), 43 deletions(-) diff --git a/classes/CCR/DB/MySQLDB.php b/classes/CCR/DB/MySQLDB.php index 9d90671082..ef4e4f60a9 100644 --- a/classes/CCR/DB/MySQLDB.php +++ b/classes/CCR/DB/MySQLDB.php @@ -15,20 +15,18 @@ use Exception; -class MySQLDB -extends PDODB -implements iDatabase +class MySQLDB extends PDODB implements iDatabase { // ------------------------------------------------------------------------------------------ // @see iDatabase::__construct() // ------------------------------------------------------------------------------------------ - function __construct($db_host, $db_port, $db_name, $db_username, $db_password, $dsn_extra = null) - { + public function __construct($db_host, $db_port, $db_name, $db_username, $db_password, $dsn_extra = null) + { if ( null == $db_host || null === $db_name || null === $db_username ) { $msg = "Database engine " . __CLASS__ . " requires (host, database, username)"; throw new Exception($msg); } - parent::__construct("mysql", $db_host, $db_port, $db_name, $db_username, $db_password, 'charset=utf8'); - } + parent::__construct("mysql", $db_host, $db_port, $db_name, $db_username, $db_password, 'charset=utf8'); + } } // class MySQLDB diff --git a/classes/CCR/DB/NullDB.php b/classes/CCR/DB/NullDB.php index 7d31062045..26082ad839 100644 --- a/classes/CCR/DB/NullDB.php +++ b/classes/CCR/DB/NullDB.php @@ -14,7 +14,7 @@ class NullDB implements iDatabase { - public function __construct($db_engine, $db_host, $db_port, $db_name, $db_username, $db_password) + public function __construct() { } diff --git a/classes/CCR/DB/PostgresDB.php b/classes/CCR/DB/PostgresDB.php index bdd100491e..f05436ccab 100644 --- a/classes/CCR/DB/PostgresDB.php +++ b/classes/CCR/DB/PostgresDB.php @@ -1,36 +1,34 @@ -* - Now implements iDatabase -* -*/ + * @author Amin Ghadersohi + * @date 2010-Jul-07 + * + * The top interface for postgres dbs using pdo driver + * + * Changelog + * + * 2015-12-15 Steve Gallo + * - Now implements iDatabase + * + */ namespace CCR\DB; use Exception; -class PostgresDB -extends PDODB -implements iDatabase +class PostgresDB extends PDODB implements iDatabase { // ------------------------------------------------------------------------------------------ // @see iDatabase::__construct() // ------------------------------------------------------------------------------------------ - function __construct($db_host, $db_port, $db_name, $db_username, $db_password, $dsn_extra = null) - { + public function __construct($db_host, $db_port, $db_name, $db_username, $db_password, $dsn_extra = null) + { if ( null == $db_host || null === $db_name || null === $db_username ) { $msg = "Database engine " . __CLASS__ . " requires (host, database, username)"; throw new Exception($msg); } - parent::__construct("pgsql", $db_host, $db_port, $db_name, $db_username, $db_password); - } + parent::__construct("pgsql", $db_host, $db_port, $db_name, $db_username, $db_password); + } } // class PostgresDB diff --git a/classes/CCR/DB/iDatabase.php b/classes/CCR/DB/iDatabase.php index 0f99f6b71a..f453d40649 100644 --- a/classes/CCR/DB/iDatabase.php +++ b/classes/CCR/DB/iDatabase.php @@ -12,8 +12,7 @@ * - Changed name from Database to iDatabase for consistency with coding conventions * * 2017-01-12 Steve Gallo - * - Added methods that were added over time to PDODB (__construct(), handle(), insert(), - * getRowCount()) + * - Added methods that were added over time to PDODB (handle(), insert(), getRowCount()) * - Added documentation and cleaned up for consistency * ========================================================================================== */ @@ -22,21 +21,6 @@ interface iDatabase { - /* ------------------------------------------------------------------------------------------ - * Constructor - * - * @param $db_engine PDO database engine name - * @param $db_host Database hostname - * @param $db_port Database port - * @param $db_name Database name - * @param $db_username Database username - * @param $db_password Database user password - * @param $dsn_extra Optional extra parameters to be added to the DSN - * ------------------------------------------------------------------------------------------ - */ - - public function __construct($db_engine, $db_host, $db_port, $db_name, $db_username, $db_password); - /* ------------------------------------------------------------------------------------------ * Perform any necessary cleanup when the object is destroyed, such as closing the * connection From 2ea92dce50b3b753690792d375217370fbc8e021 Mon Sep 17 00:00:00 2001 From: Steve Gallo Date: Sat, 21 Jan 2017 12:46:52 -0500 Subject: [PATCH 14/15] Remove underscores from private class members --- classes/CCR/DB/PDODB.php | 84 ++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/classes/CCR/DB/PDODB.php b/classes/CCR/DB/PDODB.php index ad9127d7dd..8f148ab246 100644 --- a/classes/CCR/DB/PDODB.php +++ b/classes/CCR/DB/PDODB.php @@ -29,25 +29,25 @@ class PDODB implements iDatabase { // Database connection parameters - public $_db_engine = null; - public $_db_host = null; - public $_db_port = null; - public $_db_name = null; - public $_db_username = null; - public $_db_password = null; + public $db_engine = null; + public $db_host = null; + public $db_port = null; + public $db_name = null; + public $db_username = null; + public $db_password = null; // Optional extra parameters to be added to the DSN - public $_dsn_extra = null; + public $dsn_extra = null; // Generated DSN - protected $_dsn = null; + protected $dsn = null; // Handle to the PDO instance - protected $_dbh = null; + protected $dbh = null; - protected static $_debug_mode = false; - protected static $_queries = array(); - protected static $_params = array(); + protected static $debug_mode = false; + protected static $queries = array(); + protected static $params = array(); // -------------------------------------------------------------------------------- // @see iDatabase::__construct() @@ -55,13 +55,13 @@ class PDODB implements iDatabase public function __construct($db_engine, $db_host, $db_port, $db_name, $db_username, $db_password, $dsn_extra = null) { - $this->_db_engine = $db_engine; - $this->_db_host = $db_host; - $this->_db_port = $db_port; - $this->_db_name = $db_name; - $this->_db_username = $db_username; - $this->_db_password = $db_password; - $this->_dsn_extra = $dsn_extra; + $this->db_engine = $db_engine; + $this->db_host = $db_host; + $this->db_port = $db_port; + $this->db_name = $db_name; + $this->db_username = $db_username; + $this->db_password = $db_password; + $this->dsn_extra = $dsn_extra; } // __construct() // -------------------------------------------------------------------------------- @@ -79,15 +79,15 @@ public function __destruct() public function connect() { - if ( null !== $this->_dbh) { - return $this->_dbh; + if ( null !== $this->dbh) { + return $this->dbh; } - $this->_dsn = $this->generateDsn(); - $this->_dbh = new PDO($this->_dsn, $this->_db_username, $this->_db_password); - $this->_dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); + $this->dsn = $this->generateDsn(); + $this->dbh = new PDO($this->dsn, $this->db_username, $this->db_password); + $this->dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); - return $this->_dbh; + return $this->dbh; } // connect() // -------------------------------------------------------------------------------- @@ -96,7 +96,7 @@ public function connect() public function disconnect() { - $this->_dbh = null; + $this->dbh = null; } @@ -117,13 +117,13 @@ public function handle() protected function generateDsn() { - $dsn = $this->_db_engine - . ':host=' . $this->_db_host - . ( null !== $this->_db_port ? ';port=' . $this->_db_port : '' ) - . ';dbname=' . $this->_db_name; + $dsn = $this->db_engine + . ':host=' . $this->db_host + . ( null !== $this->db_port ? ';port=' . $this->db_port : '' ) + . ';dbname=' . $this->db_name; - if ( null !== $this->_dsn_extra ) { - $dsn .= ( 0 !== strpos($this->_dsn_extra, ';') ? ';' : '' ) . $this->_dsn_extra; + if ( null !== $this->dsn_extra ) { + $dsn .= ( 0 !== strpos($this->dsn_extra, ';') ? ';' : '' ) . $this->dsn_extra; } return $dsn; @@ -135,7 +135,7 @@ protected function generateDsn() public function getDsn() { - return $this->_dsn; + return $this->dsn; } // getDsn() // -------------------------------------------------------------------------------- @@ -304,25 +304,25 @@ public function quote($string) public static function debugInfo() { return array( - "queries" => PDODB::$_queries, - "params" => PDODB::$_params + "queries" => PDODB::$queries, + "params" => PDODB::$params ); } public static function debugOn() { - PDODB::$_debug_mode = true; + PDODB::$debug_mode = true; } public static function debugOff() { - PDODB::$_debug_mode = false; + PDODB::$debug_mode = false; } public static function resetDebugInfo() { - unset($GLOBALS['PDODB::$_queries']); - unset($GLOBALS['PDODB::$_params']); + unset($GLOBALS['PDODB::$queries']); + unset($GLOBALS['PDODB::$params']); } private function debugging() @@ -334,13 +334,13 @@ private function debugging() } catch (Exception $e) { } - return $sql_debug_mode || PDODB::$_debug_mode; + return $sql_debug_mode || PDODB::$debug_mode; } private function debug($query, $params) { - PDODB::$_queries[] = trim(preg_replace("(\s+)", " ", $query)); - PDODB::$_params[] = PDODB::protectParams($params); + PDODB::$queries[] = trim(preg_replace("(\s+)", " ", $query)); + PDODB::$params[] = PDODB::protectParams($params); } private static function protectParams($params) From 3cc0fc454e727a43bd707175b79f724e1800fc8a Mon Sep 17 00:00:00 2001 From: Steve Gallo Date: Sun, 22 Jan 2017 14:55:28 -0500 Subject: [PATCH 15/15] Revert composer updates --- composer.lock | 348 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 241 insertions(+), 107 deletions(-) diff --git a/composer.lock b/composer.lock index 0c6810754e..2af02a00ec 100644 --- a/composer.lock +++ b/composer.lock @@ -503,23 +503,23 @@ }, { "name": "justinrainbow/json-schema", - "version": "2.0.5", + "version": "2.0.3", "source": { "type": "git", "url": "https://github.com/justinrainbow/json-schema.git", - "reference": "6b2a33e6a768f96bdc2ead5600af0822eed17d67" + "reference": "c21534c635f03428e92254333fab4ae35b2cdfd9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/justinrainbow/json-schema/zipball/6b2a33e6a768f96bdc2ead5600af0822eed17d67", - "reference": "6b2a33e6a768f96bdc2ead5600af0822eed17d67", + "url": "https://api.github.com/repos/justinrainbow/json-schema/zipball/c21534c635f03428e92254333fab4ae35b2cdfd9", + "reference": "c21534c635f03428e92254333fab4ae35b2cdfd9", "shasum": "" }, "require": { "php": ">=5.3.3" }, "require-dev": { - "json-schema/json-schema-test-suite": "1.2.0", + "json-schema/json-schema-test-suite": "1.1.2", "phpdocumentor/phpdocumentor": "~2", "phpunit/phpunit": "^4.8.22" }, @@ -565,7 +565,7 @@ "json", "schema" ], - "time": "2016-06-02T10:59:52+00:00" + "time": "2016-05-10T20:38:51+00:00" }, { "name": "moment/moment-min-file", @@ -706,30 +706,22 @@ }, { "name": "psr/log", - "version": "1.0.2", + "version": "1.0.0", "source": { "type": "git", "url": "https://github.com/php-fig/log.git", - "reference": "4ebe3a8bf773a19edfe0a84b6585ba3d401b724d" + "reference": "fe0936ee26643249e916849d48e3a51d5f5e278b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-fig/log/zipball/4ebe3a8bf773a19edfe0a84b6585ba3d401b724d", - "reference": "4ebe3a8bf773a19edfe0a84b6585ba3d401b724d", + "url": "https://api.github.com/repos/php-fig/log/zipball/fe0936ee26643249e916849d48e3a51d5f5e278b", + "reference": "fe0936ee26643249e916849d48e3a51d5f5e278b", "shasum": "" }, - "require": { - "php": ">=5.3.0" - }, "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.0.x-dev" - } - }, "autoload": { - "psr-4": { - "Psr\\Log\\": "Psr/Log/" + "psr-0": { + "Psr\\Log\\": "" } }, "notification-url": "https://packagist.org/downloads/", @@ -743,26 +735,25 @@ } ], "description": "Common interface for logging libraries", - "homepage": "https://github.com/php-fig/log", "keywords": [ "log", "psr", "psr-3" ], - "time": "2016-10-10T12:19:37+00:00" + "time": "2012-12-21T11:40:51+00:00" }, { "name": "robrichards/xmlseclibs", - "version": "1.4.2", + "version": "1.4.1", "source": { "type": "git", "url": "https://github.com/robrichards/xmlseclibs.git", - "reference": "79fb5e03c4ee4dc3ec77e4b2628231374364a017" + "reference": "465f18a8e1196c279b1298a3b08bcbee71ea4e4e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/robrichards/xmlseclibs/zipball/79fb5e03c4ee4dc3ec77e4b2628231374364a017", - "reference": "79fb5e03c4ee4dc3ec77e4b2628231374364a017", + "url": "https://api.github.com/repos/robrichards/xmlseclibs/zipball/465f18a8e1196c279b1298a3b08bcbee71ea4e4e", + "reference": "465f18a8e1196c279b1298a3b08bcbee71ea4e4e", "shasum": "" }, "require": { @@ -790,7 +781,7 @@ "xml", "xmldsig" ], - "time": "2016-09-08T13:31:44+00:00" + "time": "2015-07-31T12:22:14+00:00" }, { "name": "sencha/extjs-gpl", @@ -897,16 +888,16 @@ }, { "name": "simplesamlphp/saml2", - "version": "v1.9.1", + "version": "v1.8", "source": { "type": "git", "url": "https://github.com/simplesamlphp/saml2.git", - "reference": "2a2bd4398257cbe72251c68348be72707cc77262" + "reference": "11891064b8a2d1a483925cade7a9277f36b6055e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/simplesamlphp/saml2/zipball/2a2bd4398257cbe72251c68348be72707cc77262", - "reference": "2a2bd4398257cbe72251c68348be72707cc77262", + "url": "https://api.github.com/repos/simplesamlphp/saml2/zipball/11891064b8a2d1a483925cade7a9277f36b6055e", + "reference": "11891064b8a2d1a483925cade7a9277f36b6055e", "shasum": "" }, "require": { @@ -942,34 +933,26 @@ } ], "description": "SAML2 PHP library from SimpleSAMLphp", - "time": "2016-12-02T12:45:13+00:00" + "time": "2016-01-27T14:23:34+00:00" }, { "name": "simplesamlphp/simplesamlphp", - "version": "v1.14.11", + "version": "v1.14.0", "source": { "type": "git", "url": "https://github.com/simplesamlphp/simplesamlphp.git", - "reference": "b96dcc500caae73954d4f01189e0209afc6086be" + "reference": "03c6303dfee814450836c4ee3d07d51e628e4d4e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/simplesamlphp/simplesamlphp/zipball/b96dcc500caae73954d4f01189e0209afc6086be", - "reference": "b96dcc500caae73954d4f01189e0209afc6086be", + "url": "https://api.github.com/repos/simplesamlphp/simplesamlphp/zipball/03c6303dfee814450836c4ee3d07d51e628e4d4e", + "reference": "03c6303dfee814450836c4ee3d07d51e628e4d4e", "shasum": "" }, "require": { - "ext-date": "*", - "ext-dom": "*", - "ext-hash": "*", - "ext-json": "*", - "ext-openssl": "*", - "ext-pcre": "*", - "ext-spl": "*", - "ext-zlib": "*", "php": ">=5.3", "robrichards/xmlseclibs": "~1.4.1", - "simplesamlphp/saml2": "~1.9.1", + "simplesamlphp/saml2": "~1.7", "whitehat101/apr1-md5": "~1.0" }, "require-dev": { @@ -1012,7 +995,7 @@ "sp", "ws-federation" ], - "time": "2016-12-12T15:13:53+00:00" + "time": "2016-02-15T12:18:12+00:00" }, { "name": "symfony/debug", @@ -1572,6 +1555,48 @@ ], "time": "2015-06-14T21:17:01+00:00" }, + { + "name": "ircmaxell/password-compat", + "version": "v1.0.4", + "source": { + "type": "git", + "url": "https://github.com/ircmaxell/password_compat.git", + "reference": "5c5cde8822a69545767f7c7f3058cb15ff84614c" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/ircmaxell/password_compat/zipball/5c5cde8822a69545767f7c7f3058cb15ff84614c", + "reference": "5c5cde8822a69545767f7c7f3058cb15ff84614c", + "shasum": "" + }, + "require-dev": { + "phpunit/phpunit": "4.*" + }, + "type": "library", + "autoload": { + "files": [ + "lib/password.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Anthony Ferrara", + "email": "ircmaxell@php.net", + "homepage": "http://blog.ircmaxell.com" + } + ], + "description": "A compatibility library for the proposed simplified password hashing algorithm: https://wiki.php.net/rfc/password_hash", + "homepage": "https://github.com/ircmaxell/password_compat", + "keywords": [ + "hashing", + "password" + ], + "time": "2014-11-20T16:49:30+00:00" + }, { "name": "phpdocumentor/reflection-docblock", "version": "2.0.4", @@ -1623,33 +1648,32 @@ }, { "name": "phpspec/prophecy", - "version": "v1.6.2", + "version": "v1.6.0", "source": { "type": "git", "url": "https://github.com/phpspec/prophecy.git", - "reference": "6c52c2722f8460122f96f86346600e1077ce22cb" + "reference": "3c91bdf81797d725b14cb62906f9a4ce44235972" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpspec/prophecy/zipball/6c52c2722f8460122f96f86346600e1077ce22cb", - "reference": "6c52c2722f8460122f96f86346600e1077ce22cb", + "url": "https://api.github.com/repos/phpspec/prophecy/zipball/3c91bdf81797d725b14cb62906f9a4ce44235972", + "reference": "3c91bdf81797d725b14cb62906f9a4ce44235972", "shasum": "" }, "require": { "doctrine/instantiator": "^1.0.2", "php": "^5.3|^7.0", - "phpdocumentor/reflection-docblock": "^2.0|^3.0.2", - "sebastian/comparator": "^1.1", - "sebastian/recursion-context": "^1.0|^2.0" + "phpdocumentor/reflection-docblock": "~2.0", + "sebastian/comparator": "~1.1", + "sebastian/recursion-context": "~1.0" }, "require-dev": { - "phpspec/phpspec": "^2.0", - "phpunit/phpunit": "^4.8 || ^5.6.5" + "phpspec/phpspec": "~2.0" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "1.6.x-dev" + "dev-master": "1.5.x-dev" } }, "autoload": { @@ -1682,7 +1706,7 @@ "spy", "stub" ], - "time": "2016-11-21T14:58:47+00:00" + "time": "2016-02-15T07:46:21+00:00" }, { "name": "phpunit/php-code-coverage", @@ -1748,16 +1772,16 @@ }, { "name": "phpunit/php-file-iterator", - "version": "1.4.2", + "version": "1.4.1", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-file-iterator.git", - "reference": "3cc8f69b3028d0f96a9078e6295d86e9bf019be5" + "reference": "6150bf2c35d3fc379e50c7602b75caceaa39dbf0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/3cc8f69b3028d0f96a9078e6295d86e9bf019be5", - "reference": "3cc8f69b3028d0f96a9078e6295d86e9bf019be5", + "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/6150bf2c35d3fc379e50c7602b75caceaa39dbf0", + "reference": "6150bf2c35d3fc379e50c7602b75caceaa39dbf0", "shasum": "" }, "require": { @@ -1791,7 +1815,7 @@ "filesystem", "iterator" ], - "time": "2016-10-03T07:40:28+00:00" + "time": "2015-06-21T13:08:43+00:00" }, { "name": "phpunit/php-text-template", @@ -1836,24 +1860,21 @@ }, { "name": "phpunit/php-timer", - "version": "1.0.8", + "version": "1.0.7", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-timer.git", - "reference": "38e9124049cf1a164f1e4537caf19c99bf1eb260" + "reference": "3e82f4e9fc92665fafd9157568e4dcb01d014e5b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-timer/zipball/38e9124049cf1a164f1e4537caf19c99bf1eb260", - "reference": "38e9124049cf1a164f1e4537caf19c99bf1eb260", + "url": "https://api.github.com/repos/sebastianbergmann/php-timer/zipball/3e82f4e9fc92665fafd9157568e4dcb01d014e5b", + "reference": "3e82f4e9fc92665fafd9157568e4dcb01d014e5b", "shasum": "" }, "require": { "php": ">=5.3.3" }, - "require-dev": { - "phpunit/phpunit": "~4|~5" - }, "type": "library", "autoload": { "classmap": [ @@ -1876,20 +1897,20 @@ "keywords": [ "timer" ], - "time": "2016-05-12T18:03:57+00:00" + "time": "2015-06-21T08:01:12+00:00" }, { "name": "phpunit/php-token-stream", - "version": "1.4.9", + "version": "1.4.8", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-token-stream.git", - "reference": "3b402f65a4cc90abf6e1104e388b896ce209631b" + "reference": "3144ae21711fb6cac0b1ab4cbe63b75ce3d4e8da" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-token-stream/zipball/3b402f65a4cc90abf6e1104e388b896ce209631b", - "reference": "3b402f65a4cc90abf6e1104e388b896ce209631b", + "url": "https://api.github.com/repos/sebastianbergmann/php-token-stream/zipball/3144ae21711fb6cac0b1ab4cbe63b75ce3d4e8da", + "reference": "3144ae21711fb6cac0b1ab4cbe63b75ce3d4e8da", "shasum": "" }, "require": { @@ -1925,20 +1946,20 @@ "keywords": [ "tokenizer" ], - "time": "2016-11-15T14:06:22+00:00" + "time": "2015-09-15T10:49:45+00:00" }, { "name": "phpunit/phpunit", - "version": "4.8.31", + "version": "4.8.23", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "98b2b39a520766bec663ff5b7ff1b729db9dbfe3" + "reference": "6e351261f9cd33daf205a131a1ba61c6d33bd483" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/98b2b39a520766bec663ff5b7ff1b729db9dbfe3", - "reference": "98b2b39a520766bec663ff5b7ff1b729db9dbfe3", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/6e351261f9cd33daf205a131a1ba61c6d33bd483", + "reference": "6e351261f9cd33daf205a131a1ba61c6d33bd483", "shasum": "" }, "require": { @@ -1952,9 +1973,9 @@ "phpunit/php-code-coverage": "~2.1", "phpunit/php-file-iterator": "~1.4", "phpunit/php-text-template": "~1.2", - "phpunit/php-timer": "^1.0.6", + "phpunit/php-timer": ">=1.0.6", "phpunit/phpunit-mock-objects": "~2.3", - "sebastian/comparator": "~1.2.2", + "sebastian/comparator": "~1.1", "sebastian/diff": "~1.2", "sebastian/environment": "~1.3", "sebastian/exporter": "~1.2", @@ -1997,7 +2018,7 @@ "testing", "xunit" ], - "time": "2016-12-09T02:45:31+00:00" + "time": "2016-02-11T14:56:33+00:00" }, { "name": "phpunit/phpunit-mock-objects", @@ -2057,22 +2078,22 @@ }, { "name": "sebastian/comparator", - "version": "1.2.2", + "version": "1.2.0", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/comparator.git", - "reference": "6a1ed12e8b2409076ab22e3897126211ff8b1f7f" + "reference": "937efb279bd37a375bcadf584dec0726f84dbf22" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/6a1ed12e8b2409076ab22e3897126211ff8b1f7f", - "reference": "6a1ed12e8b2409076ab22e3897126211ff8b1f7f", + "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/937efb279bd37a375bcadf584dec0726f84dbf22", + "reference": "937efb279bd37a375bcadf584dec0726f84dbf22", "shasum": "" }, "require": { "php": ">=5.3.3", "sebastian/diff": "~1.2", - "sebastian/exporter": "~1.2 || ~2.0" + "sebastian/exporter": "~1.2" }, "require-dev": { "phpunit/phpunit": "~4.4" @@ -2117,7 +2138,7 @@ "compare", "equality" ], - "time": "2016-11-19T09:18:40+00:00" + "time": "2015-07-26T15:48:44+00:00" }, { "name": "sebastian/diff", @@ -2173,23 +2194,23 @@ }, { "name": "sebastian/environment", - "version": "1.3.8", + "version": "1.3.3", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/environment.git", - "reference": "be2c607e43ce4c89ecd60e75c6a85c126e754aea" + "reference": "6e7133793a8e5a5714a551a8324337374be209df" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/be2c607e43ce4c89ecd60e75c6a85c126e754aea", - "reference": "be2c607e43ce4c89ecd60e75c6a85c126e754aea", + "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/6e7133793a8e5a5714a551a8324337374be209df", + "reference": "6e7133793a8e5a5714a551a8324337374be209df", "shasum": "" }, "require": { - "php": "^5.3.3 || ^7.0" + "php": ">=5.3.3" }, "require-dev": { - "phpunit/phpunit": "^4.8 || ^5.0" + "phpunit/phpunit": "~4.4" }, "type": "library", "extra": { @@ -2219,20 +2240,20 @@ "environment", "hhvm" ], - "time": "2016-08-18T05:49:44+00:00" + "time": "2015-12-02T08:37:27+00:00" }, { "name": "sebastian/exporter", - "version": "1.2.2", + "version": "1.2.1", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/exporter.git", - "reference": "42c4c2eec485ee3e159ec9884f95b431287edde4" + "reference": "7ae5513327cb536431847bcc0c10edba2701064e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/42c4c2eec485ee3e159ec9884f95b431287edde4", - "reference": "42c4c2eec485ee3e159ec9884f95b431287edde4", + "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/7ae5513327cb536431847bcc0c10edba2701064e", + "reference": "7ae5513327cb536431847bcc0c10edba2701064e", "shasum": "" }, "require": { @@ -2240,13 +2261,12 @@ "sebastian/recursion-context": "~1.0" }, "require-dev": { - "ext-mbstring": "*", "phpunit/phpunit": "~4.4" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "1.3.x-dev" + "dev-master": "1.2.x-dev" } }, "autoload": { @@ -2286,7 +2306,7 @@ "export", "exporter" ], - "time": "2016-06-17T09:04:28+00:00" + "time": "2015-06-21T07:55:53+00:00" }, { "name": "sebastian/global-state", @@ -2429,16 +2449,16 @@ }, { "name": "squizlabs/php_codesniffer", - "version": "2.7.1", + "version": "2.6.0", "source": { "type": "git", "url": "https://github.com/squizlabs/PHP_CodeSniffer.git", - "reference": "9b324f3a1132459a7274a0ace2e1b766ba80930f" + "reference": "1bcdf03b068a530ac1962ce671dead356eeba43b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/9b324f3a1132459a7274a0ace2e1b766ba80930f", - "reference": "9b324f3a1132459a7274a0ace2e1b766ba80930f", + "url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/1bcdf03b068a530ac1962ce671dead356eeba43b", + "reference": "1bcdf03b068a530ac1962ce671dead356eeba43b", "shasum": "" }, "require": { @@ -2503,7 +2523,121 @@ "phpcs", "standards" ], - "time": "2016-11-30T04:02:31+00:00" + "time": "2016-04-03T22:58:34+00:00" + }, + { + "name": "symfony/polyfill-php54", + "version": "v1.1.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-php54.git", + "reference": "74663d5a2ff3c530c1bc0571500e0feec9094054" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-php54/zipball/74663d5a2ff3c530c1bc0571500e0feec9094054", + "reference": "74663d5a2ff3c530c1bc0571500e0feec9094054", + "shasum": "" + }, + "require": { + "php": ">=5.3.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.1-dev" + } + }, + "autoload": { + "psr-4": { + "Symfony\\Polyfill\\Php54\\": "" + }, + "files": [ + "bootstrap.php" + ], + "classmap": [ + "Resources/stubs" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill backporting some PHP 5.4+ features to lower PHP versions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "polyfill", + "portable", + "shim" + ], + "time": "2016-01-20T09:13:37+00:00" + }, + { + "name": "symfony/polyfill-php55", + "version": "v1.1.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-php55.git", + "reference": "b4f3f07d91702f8f926339fc4fcf81671d8c27e6" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-php55/zipball/b4f3f07d91702f8f926339fc4fcf81671d8c27e6", + "reference": "b4f3f07d91702f8f926339fc4fcf81671d8c27e6", + "shasum": "" + }, + "require": { + "ircmaxell/password-compat": "~1.0", + "php": ">=5.3.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.1-dev" + } + }, + "autoload": { + "psr-4": { + "Symfony\\Polyfill\\Php55\\": "" + }, + "files": [ + "bootstrap.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill backporting some PHP 5.5+ features to lower PHP versions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "polyfill", + "portable", + "shim" + ], + "time": "2016-01-20T09:13:37+00:00" }, { "name": "symfony/yaml",