From 58b2bb84ce083d14472fbc076afece07456e84b1 Mon Sep 17 00:00:00 2001 From: Steve Gallo Date: Mon, 15 May 2017 11:32:04 -0400 Subject: [PATCH] Refactoring of ETL/DbEntity into ETL/DbModel (#138) * Added tests for DbEntity\Table * Moved namespace ETL\DbEntity to ETL\DbModel * Refactoring of ETL/DbModel to remove old code and support upcoming features * Updated commit for xdmod-test-artifacts * Fix record formula verification and saving of overseer restriction value * Add @plessbd blacklist filters * Ignore case where temp directory already exists for tests --- classes/ETL/Aggregator/aAggregator.php | 2 +- classes/ETL/Aggregator/pdoAggregator.php | 64 +- classes/ETL/DbEntity/AggregationTable.php | 277 ----- classes/ETL/DbEntity/Index.php | 232 ----- classes/ETL/DbEntity/Join.php | 193 ---- classes/ETL/DbEntity/Query.php | 860 ---------------- classes/ETL/DbEntity/Table.php | 944 ------------------ classes/ETL/DbEntity/Trigger.php | 246 ----- classes/ETL/DbEntity/aNamedEntity.php | 166 --- classes/ETL/DbEntity/iTableItem.php | 106 -- classes/ETL/DbModel/AggregationTable.php | 237 +++++ classes/ETL/{DbEntity => DbModel}/Column.php | 306 ++---- classes/ETL/DbModel/Entity.php | 503 ++++++++++ classes/ETL/DbModel/Index.php | 207 ++++ classes/ETL/DbModel/Join.php | 116 +++ classes/ETL/DbModel/NamedEntity.php | 98 ++ classes/ETL/DbModel/Query.php | 525 ++++++++++ classes/ETL/DbModel/SchemaEntity.php | 100 ++ classes/ETL/DbModel/Table.php | 833 ++++++++++++++++ classes/ETL/DbModel/Trigger.php | 162 +++ classes/ETL/DbModel/iAlterableEntity.php | 35 + classes/ETL/DbModel/iDiscoverableEntity.php | 35 + classes/ETL/DbModel/iEntity.php | 150 +++ classes/ETL/EtlOverseerOptions.php | 2 +- classes/ETL/Ingestor/RestIngestor.php | 8 +- classes/ETL/Ingestor/UpdateIngestor.php | 4 +- classes/ETL/Ingestor/pdoIngestor.php | 14 +- classes/ETL/Maintenance/ManageTables.php | 6 +- classes/ETL/Maintenance/VerifyDatabase.php | 6 +- classes/ETL/Utilities.php | 4 + classes/ETL/aRdbmsDestinationAction.php | 23 +- composer.lock | 10 +- .../jobs/jobfact_hpc_aggregation.json | 1 + .../Configuration/EtlConfigurationTest.php | 2 +- .../tests/lib/ETL/DbModel/DbModelTest.php | 396 ++++++++ .../modules/xdmod/tests/phpunit.xml.dist | 7 + tools/etl/etl_table_manager.php | 308 +++--- 37 files changed, 3715 insertions(+), 3473 deletions(-) delete mode 100644 classes/ETL/DbEntity/AggregationTable.php delete mode 100644 classes/ETL/DbEntity/Index.php delete mode 100644 classes/ETL/DbEntity/Join.php delete mode 100644 classes/ETL/DbEntity/Query.php delete mode 100644 classes/ETL/DbEntity/Table.php delete mode 100644 classes/ETL/DbEntity/Trigger.php delete mode 100644 classes/ETL/DbEntity/aNamedEntity.php delete mode 100644 classes/ETL/DbEntity/iTableItem.php create mode 100644 classes/ETL/DbModel/AggregationTable.php rename classes/ETL/{DbEntity => DbModel}/Column.php (55%) create mode 100644 classes/ETL/DbModel/Entity.php create mode 100644 classes/ETL/DbModel/Index.php create mode 100644 classes/ETL/DbModel/Join.php create mode 100644 classes/ETL/DbModel/NamedEntity.php create mode 100644 classes/ETL/DbModel/Query.php create mode 100644 classes/ETL/DbModel/SchemaEntity.php create mode 100644 classes/ETL/DbModel/Table.php create mode 100644 classes/ETL/DbModel/Trigger.php create mode 100644 classes/ETL/DbModel/iAlterableEntity.php create mode 100644 classes/ETL/DbModel/iDiscoverableEntity.php create mode 100644 classes/ETL/DbModel/iEntity.php create mode 100644 open_xdmod/modules/xdmod/tests/lib/ETL/DbModel/DbModelTest.php diff --git a/classes/ETL/Aggregator/aAggregator.php b/classes/ETL/Aggregator/aAggregator.php index 9aa2802ccf..2797bd8b80 100644 --- a/classes/ETL/Aggregator/aAggregator.php +++ b/classes/ETL/Aggregator/aAggregator.php @@ -120,7 +120,7 @@ public function execute(EtlOverseerOptions $etlOverseerOptions) // The aggregation unit must be set for the AggregationTable foreach ( $this->etlDestinationTableList as $etlTableKey => $etlTable ) { - $etlTable->setAggregationUnit($aggregationUnit); + $etlTable->aggregation_unit = $aggregationUnit; } $this->variableMap['AGGREGATION_UNIT'] = $aggregationUnit; diff --git a/classes/ETL/Aggregator/pdoAggregator.php b/classes/ETL/Aggregator/pdoAggregator.php index 795d1ce387..4583010f14 100644 --- a/classes/ETL/Aggregator/pdoAggregator.php +++ b/classes/ETL/Aggregator/pdoAggregator.php @@ -62,16 +62,16 @@ use ETL\aOptions; use ETL\EtlOverseerOptions; use ETL\DataEndpoint\Mysql; -use ETL\DbEntity\AggregationTable; -use ETL\DbEntity\Query; -use ETL\DbEntity\Table; +use ETL\DbModel\AggregationTable; +use ETL\DbModel\Query; +use ETL\DbModel\Table; use ETL\Utilities; use ETL\Configuration\EtlConfiguration; -use \Log; -use \PDOException; +use Log; +use PDOException; use PDOStatement; -use \PDO; +use PDO; class pdoAggregator extends aAggregator { @@ -106,8 +106,7 @@ class pdoAggregator extends aAggregator * * @param IngestorOptions $options Options specific to this Ingestor * @param EtlConfiguration $etlConfig Parsed configuration options for this ETL - * @param string $defaultTablePrefix Default table prefix as defined in the child class (e.g., - * "jobfact_by_") + * @param Log $logger PEAR Log object for system logging * ------------------------------------------------------------------------------------------ */ @@ -208,14 +207,14 @@ public function initialize(EtlOverseerOptions $etlOverseerOptions = null) // but it doesn't matter because the naming will still be consistent. $columnNames = $this->etlDestinationTable->getColumnNames(); - $missingColumnNames = array_diff($this->etlSourceQuery->getGroupBys(), $columnNames); + $missingColumnNames = array_diff($this->etlSourceQuery->groupby, $columnNames); if ( 0 != count($missingColumnNames) ) { $msg = "Columns in group by not found in table: " . implode(", ", $missingColumnNames); $this->logAndThrowException($msg); } - $missingColumnNames = array_diff(array_keys($this->etlSourceQuery->getRecords()), $columnNames); + $missingColumnNames = array_diff(array_keys($this->etlSourceQuery->records), $columnNames); if ( 0 != count($missingColumnNames) ) { $msg = "Columns in formulas not found in table: " . implode(", ", $missingColumnNames); @@ -271,18 +270,18 @@ protected function createDestinationTableObjects() $this->destinationEndpoint->getSystemQuoteChar(), $this->logger ); - $this->etlDestinationTable->setSchema($this->destinationEndpoint->getSchema()); + $this->etlDestinationTable->schema = $this->destinationEndpoint->getSchema(); if ( isset($this->options->table_prefix) && - $this->options->table_prefix != $this->etlDestinationTable->getTablePrefix() ) + $this->options->table_prefix != $this->etlDestinationTable->table_prefix ) { $msg = "Overriding table prefix from " . - $this->etlDestinationTable->getTablePrefix() + $this->etlDestinationTable->table_prefix . " to " . $this->options->table_prefix; $this->logger->debug($msg); - $this->etlDestinationTable->setTablePrefix($this->options->table_prefix); + $this->etlDestinationTable->table_prefix = $this->options->table_prefix; } // Aggregation does not support multiple destination tables but we must still populate @@ -302,7 +301,7 @@ protected function performPreExecuteTasks() { // To support programmatic manipulation of the source Query object, save off the description // of the first join (from) table - $sourceJoins = $this->etlSourceQuery->getJoins(); + $sourceJoins = $this->etlSourceQuery->joins; $this->etlSourceQueryOrigFromTable = array_shift($sourceJoins); $this->etlSourceQueryModified = false; @@ -355,7 +354,7 @@ protected function performPreAggregationUnitTasks($aggregationUnit) $this->manageTable($substitutedEtlAggregationTable, $this->destinationEndpoint); - if ( $this->options->disable_keys && "myisam" == strtolower($etlTable->getEngine()) ) { + if ( $this->options->disable_keys && "myisam" == strtolower($etlTable->engine) ) { $this->logger->info("Disable keys on $qualifiedDestTableName"); $sqlList[] = "ALTER TABLE $qualifiedDestTableName DISABLE KEYS"; } @@ -384,7 +383,7 @@ protected function performPostAggregationUnitTasks($aggregationUnit, $numAggrega $sqlList[] = "OPTIMIZE TABLE $qualifiedDestTableName"; } - if ( $this->options->disable_keys && "myisam" == strtolower($etlTable->getEngine()) ) { + if ( $this->options->disable_keys && "myisam" == strtolower($etlTable->engine) ) { $sqlList[] = "ALTER TABLE $qualifiedDestTableName ENABLE KEYS"; } } @@ -463,7 +462,7 @@ protected function getDirtyAggregationPeriods($aggregationUnit) $firstTable = $this->etlSourceQueryOrigFromTable; - $tableName = $this->sourceEndpoint->quoteSystemIdentifier($firstTable->getName()); + $tableName = $this->sourceEndpoint->quoteSystemIdentifier($firstTable->name); $aggregationPeriodQueryOptions = ( isset($this->parsedDefinitionFile->aggregation_period_query) ? $this->parsedDefinitionFile->aggregation_period_query @@ -511,13 +510,14 @@ protected function getDirtyAggregationPeriods($aggregationUnit) ); $this->logger->debug("Discover table $fromTable"); - $firstTableDef = Table::discover($fromTable, $this->sourceEndpoint, null, $this->logger); + + $firstTableDef = new Table(null, null, $this->logger); // If we are in dryrun mode the table may not have been created yet but we still want to // be able to display the generated queries so simply set the start and end day id // fields. - if ( false === $firstTableDef ) { + if ( false === $firstTableDef->discover($fromTable, $this->sourceEndpoint) ) { if ( $this->getEtlOverseerOptions()->isDryrun() ) { $startDayIdField = "start_day_id"; $endDayIdField = "end_day_id"; @@ -641,7 +641,7 @@ protected function getDirtyAggregationPeriods($aggregationUnit) ), 'joins' => array( (object) array( - 'name' => $firstTable->getName(), + 'name' => $firstTable->name, 'schema' => $this->sourceEndpoint->getSchema() ) ) @@ -658,7 +658,7 @@ protected function getDirtyAggregationPeriods($aggregationUnit) $recordRangeQuery = new Query($query, $this->sourceEndpoint->getSystemQuoteChar()); $this->getEtlOverseerOptions()->applyOverseerRestrictions($recordRangeQuery, $this->utilityEndpoint, $this); - $minMaxJoin = "( " . $recordRangeQuery->getSelectSql() . " ) record_ranges"; + $minMaxJoin = "( " . $recordRangeQuery->getSql() . " ) record_ranges"; $dateRangeRestrictionSql = "d.id BETWEEN record_ranges.start_period_id AND record_ranges.end_period_id"; } // else ( $this->getEtlOverseerOptions()->isForce() ) @@ -767,11 +767,11 @@ protected function _execute($aggregationUnit) // Remove the first join (from) and replace it with the temporary table that we are // going to create - $sourceJoins = $this->etlSourceQuery->getJoins(); + $sourceJoins = $this->etlSourceQuery->joins; $firstJoin = array_shift($sourceJoins); $newFirstJoin = clone $firstJoin; $newFirstJoin->setName($tmpTableName); - $newFirstJoin->setSchema($this->sourceEndpoint->getSchema()); + $newFirstJoin->schema = $this->sourceEndpoint->getSchema(); $this->etlSourceQuery->deleteJoins(); $this->etlSourceQuery->addJoin($newFirstJoin); @@ -786,7 +786,7 @@ protected function _execute($aggregationUnit) // We are not optimizing but have previously, restore the original FROM clause - $sourceJoins = $this->etlSourceQuery->getJoins(); + $sourceJoins = $this->etlSourceQuery->joins; array_shift($sourceJoins); $this->etlSourceQuery->deleteJoins(); $this->etlSourceQuery->addJoin($this->etlSourceQueryOrigFromTable); @@ -889,7 +889,7 @@ protected function _execute($aggregationUnit) $aggregationPeriodListOffset = 0; $done = false; - $sourceJoins = $this->etlSourceQuery->getJoins(); + $sourceJoins = $this->etlSourceQuery->joins; $firstJoin = current($sourceJoins); $tmpTableAlias = $firstJoin->getAlias(); @@ -940,7 +940,7 @@ protected function _execute($aggregationUnit) $origTableName = $this->sourceEndpoint->getSchema(true) . "." - . $this->sourceEndpoint->quoteSystemIdentifier($this->etlSourceQueryOrigFromTable->getName()); + . $this->sourceEndpoint->quoteSystemIdentifier($this->etlSourceQueryOrigFromTable->name); try { // Use the where clause from the aggregation query to create the temporary table @@ -1280,7 +1280,7 @@ protected function buildSqlStatements($aggregationUnit, $includeSchema = true) // *** Should this functionality be included in the Query itself? *** - $sourceRecords = $this->etlSourceQuery->getRecords(); + $sourceRecords = $this->etlSourceQuery->records; $substitutedRecordNames = array(); $duplicateRecords = array(); @@ -1307,18 +1307,18 @@ protected function buildSqlStatements($aggregationUnit, $includeSchema = true) } } - $this->selectSql = $this->etlSourceQuery->getSelectSql($includeSchema); + $this->selectSql = $this->etlSourceQuery->getSql($includeSchema); $this->insertSql = "INSERT INTO " . $this->etlDestinationTable->getFullName($includeSchema) . "\n" . "(" - . implode(",\n", array_keys($this->etlSourceQuery->getRecords())) + . implode(",\n", array_keys($this->etlSourceQuery->records)) . ")\nVALUES\n(" - . implode(",\n", Utilities::createPdoBindVarsFromArrayKeys($this->etlSourceQuery->getRecords())) + . implode(",\n", Utilities::createPdoBindVarsFromArrayKeys($this->etlSourceQuery->records)) . ")"; $this->optimizedInsertSql = "INSERT INTO " . $this->etlDestinationTable->getFullName($includeSchema) . "\n" . "(" . - implode(",\n", array_keys($this->etlSourceQuery->getRecords())) + implode(",\n", array_keys($this->etlSourceQuery->records)) . ")\n" . $this->selectSql; diff --git a/classes/ETL/DbEntity/AggregationTable.php b/classes/ETL/DbEntity/AggregationTable.php deleted file mode 100644 index 1dbb30f4b8..0000000000 --- a/classes/ETL/DbEntity/AggregationTable.php +++ /dev/null @@ -1,277 +0,0 @@ - - * @date 2015-11-20 - * ========================================================================================== - */ - -namespace ETL\DbEntity; - -use ETL\Utilities; -use \Exception; -use \Log; - -class AggregationTable extends Table -{ - // Aggregation unit to use when generating the SQL to populate the table - protected $aggregationUnit = null; - - // Table prefix used to generate the name along with the aggregation unit - protected $tablePrefix = null; - - // Query object for populating the table - protected $query = null; - - /* ------------------------------------------------------------------------------------------ - * Construct a table object from a JSON definition file or a definition object. The definition - * must contain, at a minimum, name and columns properties. - * - * @param $filename A filename for the JSON definition file - * or - * @param $definitionObject An object containing the table definition - * - * Optional 2nd and 3rd arguments: - * - * @param $variableMap An associative array specifying variables that will be substituted in the - * table DDL and the aggregation SELECT statement - * @param $macroDir The directory where macro files are found. - * ------------------------------------------------------------------------------------------ - */ - - public function __construct($config, $systemQuoteChar = null, Log $logger = null) - { - parent::__construct($config, $systemQuoteChar, $logger); - - $this->setTablePrefix($config->name); - - if ( isset($this->config->query) ) { - $this->query = new Query($config->query); - } - - } // __construct() - - /* ------------------------------------------------------------------------------------------ - * Return the table name. This combines the prefix and aggregation unit and is set when the - * aggregation unit is applied. - * - * @param $quote true to wrap the name in quotes to handle special characters - * - * @return The name of this table, optionally quoted - * - * @throw Exception if the aggregation unit has not been set - * ------------------------------------------------------------------------------------------ - */ - - public function getName($quote = false) - { - if ( null === $this->aggregationUnit ) { - $msg = "Aggregation unit must be set to generate table name"; - $this->logAndThrowException($msg); - } - - return parent::getName($quote); - - } // getName() - - /* ------------------------------------------------------------------------------------------ - * @return The query object used to populate the aggregation table. - * ------------------------------------------------------------------------------------------ - */ - - public function getQuery() - { - return $this->query; - } // getQuery() - - /* ------------------------------------------------------------------------------------------ - * Set the table prefix used to generate the name of the aggregation table. This is typically - * combined with the aggregation unit. - * - * @param $tablePrefix The table prefix to use when generating the table name. - * - * @return This object to support method chaining. - * - * @throws Exception if the table prefix is empty or null - * ------------------------------------------------------------------------------------------ - */ - - public function setTablePrefix($tablePrefix) - { - if ( empty($tablePrefix) ) { - $msg = "Table prefix unit cannot be empty or null"; - $this->logAndThrowException($msg); - } - - $this->tablePrefix = $tablePrefix; - return $this; - } // setTablePrefix() - - /* ------------------------------------------------------------------------------------------ - * @return The table prefix - * ------------------------------------------------------------------------------------------ - */ - - public function getTablePrefix() - { - return $this->tablePrefix; - } // getTablePrefix() - - /* ------------------------------------------------------------------------------------------ - * Set the aggregation unit used to generate the name of the aggregation table. This is typically - * combined with the table prefix. - * - * @param $aggregationUnit The aggregation unit to use when generating the table name and SQL - * - * @return This object to support method chaining. - * - * @throws Exception if the table prefix is empty or null - * ------------------------------------------------------------------------------------------ - */ - - public function setAggregationUnit($aggregationUnit) - { - if ( empty($aggregationUnit) ) { - $msg = "Aggregation unit cannot be empty or null"; - $this->logAndThrowException($msg); - } - - $this->aggregationUnit = $aggregationUnit; - - // Update the table name with the prefix + aggregation unit - - $this->name = $this->getTablePrefix() . $aggregationUnit; - - return $this; - } // setAggregationUnit() - - /* ------------------------------------------------------------------------------------------ - * @return The aggregation unit - * ------------------------------------------------------------------------------------------ - */ - - public function getAggregationUnit() - { - return $this->aggregationUnit; - } // getAggregationUnit() - - /* ------------------------------------------------------------------------------------------ - * Generate an object representation of this item suitable for encoding into JSON. - * - * @param $succinct true to use a succinct representation. - * @param $includeSchema true to include the schema in the table definition - * - * @return An object representation for this item suitable for encoding into JSON. - * ------------------------------------------------------------------------------------------ - */ - - public function toJsonObj($succinct = false, $includeSchema = false) - { - $data = parent::toJsonObj($succinct, $includeSchema); - - $data->query = $this->query->toJsonObj($succinct, $includeSchema); - - return $data; - - } // toJsonObj() - - /* ------------------------------------------------------------------------------------------ - * Generate a JSON representation of this table. - * - * @param $succinct true if a succinct representation should be returned. - * @param $includeSchema true to include the schema in the table definition - * - * @return A JSON formatted string representing the tabe. - * ------------------------------------------------------------------------------------------ - */ - - public function toJson($succinct = false, $includeSchema = false) - { - return json_encode($this->toJsonObj($succinct, $includeSchema)); - } // toJson() - - /* ------------------------------------------------------------------------------------------ - * Aggregation tables support variables (e.g., ${AGGREGATION_UNIT) as defined by the aggregation - * machinery) in the column, index, and trigger definitions. These variables must be replaced with - * values prior to executing DDL statements and prior to comparing tables for generation of ALTER - * TABLE statements (e.g., prior to calling aDatabaseDestinationAction::manateTable(). - * - * Create a copy of the current table and perform variable substitution on all column, index, and - * trigger definitions. - * - * @return A copy (clone) of this table with variable substiution performed on the column, index, - * and trigger definition fields. - * ------------------------------------------------------------------------------------------ - */ - - public function copyAndApplyVariables(array $variableMap) - { - // Save the JSON representation for columns, indexes, triggers - - $columnJson = array(); - $indexJson = array(); - $triggerJson = array(); - - foreach ( $this->columns as $column ) { - $columnJson[] = $column->toJsonObj(); - } - foreach ( $this->indexes as $index ) { - $indexJson[] = $index->toJsonObj(); - } - foreach ( $this->triggers as $trigger ) { - $triggerJson[] = $trigger->toJsonObj(); - } - - // Clone this object and clear the existing columns, indexes, and triggers. Add them using the - // saved definitions after substitutions have been performed. - - $newTable = clone $this; - $newTable->deleteColumns()->deleteIndexes()->deleteTriggers(); - - foreach ( $columnJson as $def ) { - foreach ( $def as $key => &$value ) { - if ( null !== $variableMap ) { - $value = Utilities::substituteVariables($value, $variableMap); - } - } - unset($value); // Sever the reference with the last element - - // Add the column, allowing duplicate column names to overwrite previous values. Without - // overwrite turned on, the yearly aggregation tables with throw an exception and log a - // warning for the year_id column. - - $newTable->addColumn($def, true); - - } - - foreach ( $indexJson as $def ) { - foreach ( $def as $key => &$value ) { - if ( null !== $variableMap ) { - $value = Utilities::substituteVariables($value, $variableMap); - } - } - unset($value); // Sever the reference with the last element - $newTable->addIndex($def); - } - - foreach ( $triggerJson as $def ) { - foreach ( $def as $key => &$value ) { - if ( null !== $variableMap ) { - $value = Utilities::substituteVariables($value, $variableMap); - } - } - unset($value); // Sever the reference with the last element - $newTable->addTrigger($def); - } - - return $newTable; - - } // copyAndApplyVariables() -} // class AggregationTable diff --git a/classes/ETL/DbEntity/Index.php b/classes/ETL/DbEntity/Index.php deleted file mode 100644 index d24973e011..0000000000 --- a/classes/ETL/DbEntity/Index.php +++ /dev/null @@ -1,232 +0,0 @@ - - * @date 2015-10-29 - * - * @see Table - * @see iTableItem - * ========================================================================================== - */ - -namespace ETL\DbEntity; - -use \Log; -use \stdClass; - -class Index extends aNamedEntity implements iTableItem -{ - private $type = null; - private $is_unique = null; - private $columns = array(); - - /* ------------------------------------------------------------------------------------------ - * @see iTableItem::__construct() - * ------------------------------------------------------------------------------------------ - */ - - public function __construct($config, $systemQuoteChar = null, Log $logger = null) - { - parent::__construct($systemQuoteChar, $logger); - - if ( ! is_object($config) ) { - $msg = __CLASS__ . ": Index definition must be an array or object"; - $this->logAndThrowException($msg); - } - - $requiredKeys = array("columns"); - $this->verifyRequiredConfigKeys($requiredKeys, $config); - - $this->initialize($config); - - } // __construct() - - /* ------------------------------------------------------------------------------------------ - * @see aNamedEntity::initialize() - * ------------------------------------------------------------------------------------------ - */ - - public function initialize(stdClass $config, $force = false) - { - if ( $this->initialized && ! $force ) { - return true; - } - - if ( ! is_array($config->columns) || 0 == count($config->columns) ) { - $msg = "Index columns must be an non-empty array"; - $this->logAndThrowException($msg); - } - - if ( ! isset($config->name) ) { - $config->name = $this->generateIndexName($config->columns); - } - - foreach ( $config as $property => $value ) { - - if ( ! property_exists($this, $property) ) { - $msg = "Property '$property' in config is not supported"; - $this->logAndThrowException($msg); - } - - $this->$property = $value; - - } // foreach ( $config as $property => $value ) - - $this->initialized = true; - - } // initialize() - - /* ------------------------------------------------------------------------------------------ - * Auto-generate an index name based on the columns included in the index. If the length of the - * index name would be too large use a hash. - * - * @param $columns The array of index column names - * - * @return The generated index name - * ------------------------------------------------------------------------------------------ - */ - - private function generateIndexName(array $columns) - { - $str = implode("_", $columns); - $name = ( strlen($str) <= 32 ? $str : md5($str) ); - return "index_" . $name; - } // generateIndexName() - - /* ------------------------------------------------------------------------------------------ - * @return The list of column names for this index - * ------------------------------------------------------------------------------------------ - */ - - public function getColumnNames() - { - return $this->columns; - } // getColumnNames() - - /* ------------------------------------------------------------------------------------------ - * @return The index type, or null if no type was specified. - * ------------------------------------------------------------------------------------------ - */ - - public function getType() - { - return $this->type; - } // getType() - - /* ------------------------------------------------------------------------------------------ - * @return true if the index is unique, false if it is not, or null if not specified. - * ------------------------------------------------------------------------------------------ - */ - - public function isUnique() - { - return $this->is_unique; - } // isUnique() - - /* ------------------------------------------------------------------------------------------ - * @see iTableItem::compare() - * ------------------------------------------------------------------------------------------ - */ - - public function compare(iTableItem $cmp) - { - if ( ! $cmp instanceof Index ) { - return 1; - } - - // Indexes are considered equal if all non-null properties are the same but only the name and - // columns are required but if the type and uniqueness are provided use those in the comparison - // as well. - - if ( $this->getName() != $cmp->getName() - || $this->getColumnNames() != $cmp->getColumnNames() ) { - return -1; - } - - // The following properties have a default set by the database. If the property is not specified - // a value will be provided when the database information schema is queried. - - if ( ( null !== $this->getType() && null !== $cmp->getType() ) - && $this->getType() != $cmp->getType() ) { - return -11; - } - - // The following properties do not have defaults set by the database and should be considered if - // one of them is set. - - // By default a primary key in MySQL has the name PRIMARY and is unique - - if ( "PRIMARY" != $this->getName() && "PRIMARY" != $cmp->getName() - && ( null !== $this->isUnique() && null !== $cmp->isUnique() ) - && $this->isUnique() != $cmp->isUnique() ) { - return -111; - } - - return 0; - - } // compare() - - /* ------------------------------------------------------------------------------------------ - * @see iTableItem::getCreateSql() - * ------------------------------------------------------------------------------------------ - */ - - public function getCreateSql($includeSchema = false) - { - // Primary keys always have an index name of "PRIMARY" - // See https://dev.mysql.com/doc/refman/5.7/en/create-table.html - - // Indexes may be created or altered in different ways (CREATE TABLE vs. ALTER TABLE) so we only - // return the essentials of the definition and let the Table class figure out the appropriate - // way to put them together. - - $parts = array(); - $parts[] = (null !== $this->name && "PRIMARY" == $this->name - ? "PRIMARY KEY" - : ( null !== $this->is_unique && $this->is_unique ? "UNIQUE ": "") . "INDEX " . $this->getName(true) ); - if ( null !== $this->type ) { - $parts[] = "USING {$this->type}"; - } - $parts[] = "(" . implode(", ", array_map(array($this, 'quote'), $this->columns)) . ")"; - - return implode(" ", $parts); - - } // getCreateSql() - - /* ------------------------------------------------------------------------------------------ - * @see iTableItem::getAlterSql() - * ------------------------------------------------------------------------------------------ - */ - - public function getAlterSql($includeSchema = false) - { - return $this->getCreateSql($includeSchema); - } // getAlterSql() - - /* ------------------------------------------------------------------------------------------ - * @see iTableItem::toJsonObj() - * ------------------------------------------------------------------------------------------ - */ - - public function toJsonObj($succinct = false) - { - if ( $succinct ) { - $data = $this->columns; - } else { - $data = new stdClass; - $data->name = $this->name; - $data->columns = $this->columns; - if ( null !== $this->type ) { - $data->type = $this->type; - } - if ( null !== $this->is_unique ) { - $data->is_unique = ( 1 == $this->is_unique); - } - } - - return $data; - - } // toJsonObj() -} // class Index diff --git a/classes/ETL/DbEntity/Join.php b/classes/ETL/DbEntity/Join.php deleted file mode 100644 index d6f5c4b0f6..0000000000 --- a/classes/ETL/DbEntity/Join.php +++ /dev/null @@ -1,193 +0,0 @@ - - * @date 2015-10-29 - * - * @see Table - * @see iTableItem - * ========================================================================================== - */ - -namespace ETL\DbEntity; - -use \Log; -use \stdClass; - -class Join extends aNamedEntity implements iTableItem -{ - // NOTE: The join name is treated as the table. - - // Optional join type (e.g., "LEFT OUTER") - private $type = null; - - // Alias for the joined table - private $alias = null; - - // Optional ON clause - private $on = null; - - /* ------------------------------------------------------------------------------------------ - * @see iTableItem::__construct() - * ------------------------------------------------------------------------------------------ - */ - - public function __construct($config, $systemQuoteChar = null, Log $logger = null) - { - parent::__construct($systemQuoteChar, $logger); - - if ( ! is_object($config) ) { - $msg = __CLASS__ . ": Join definition must be an object"; - $this->logAndThrowException($msg); - } - - $requiredKeys = array("name"); - $this->verifyRequiredConfigKeys($requiredKeys, $config); - - $this->initialize($config); - - } // __construct() - - /* ------------------------------------------------------------------------------------------ - * @see aNamedEntity::initialize() - * ------------------------------------------------------------------------------------------ - */ - - public function initialize(stdClass $config, $force = false) - { - if ( $this->initialized && ! $force ) { - return true; - } - - foreach ( $config as $property => $value ) { - if ( $this->isComment($property) ) { - continue; - } - - if ( ! property_exists($this, $property) ) { - $msg = "Property '$property' in config is not supported"; - $this->logAndThrowException($msg); - } - - $this->$property = $value; - } // foreach ( $config as $property => $value ) - - $this->initialized = true; - - } // initialize() - - /* ------------------------------------------------------------------------------------------ - * Return the optional ON clause for this join. - * - * @return The on clause, or null if no on clause was specified. - * ------------------------------------------------------------------------------------------ - */ - - public function getType() - { - return $this->type; - } // getType() - - /* ------------------------------------------------------------------------------------------ - * Return the optional alias for this table table. - * - * @return The alias, or null if no alias was specified. - * ------------------------------------------------------------------------------------------ - */ - - public function getAlias() - { - return $this->alias; - } // getAlias() - - /* ------------------------------------------------------------------------------------------ - * Return the optional ON clause for this join. - * - * @return The on clause, or null if no on clause was specified. - * ------------------------------------------------------------------------------------------ - */ - - public function getOn() - { - return $this->on; - } // getOn() - - /* ------------------------------------------------------------------------------------------ - * @see iTableItem::compare() - * ------------------------------------------------------------------------------------------ - */ - - public function compare(iTableItem $cmp) - { - if ( ! $cmp instanceof Join ) { - return 1; - } - - return 0; - - } // compare() - - /* ------------------------------------------------------------------------------------------ - * @see iTableItem::getCreateSql() - * ------------------------------------------------------------------------------------------ - */ - - public function getCreateSql($includeSchema = false) - { - $parts = array(); - - // Allow subqueries to be included and not quoted - $quoteName = ( 0 !== strpos($this->getName(), '(') ); - - $parts[] = ( null !== $this->schema && $includeSchema ? $this->getFullName() : $this->getName($quoteName) ); - if ( null !== $this->alias ) { - $parts[] = "AS {$this->alias}"; - } - if ( null !== $this->on ) { - $parts[] = "ON {$this->on}"; - } - - return implode(" ", $parts); - - } // getCreateSql() - - /* ------------------------------------------------------------------------------------------ - * @see iTableItem::getAlterSql() - * - * There is no alter SQL for this item. - * ------------------------------------------------------------------------------------------ - */ - - public function getAlterSql($includeSchema = false) - { - return ""; - } // getAlterSql() - - /* ------------------------------------------------------------------------------------------ - * @see iTableItem::toJsonObj() - * ------------------------------------------------------------------------------------------ - */ - - public function toJsonObj($succinct = false) - { - $data = new stdClass; - $data->name = $this->name; - if ( null !== $this->schema ) { - $data->schema = $this->schema; - } - if ( null !== $this->type ) { - $data->type = $this->type; - } - if ( null !== $this->alias ) { - $data->alias = $this->alias; - } - if ( null !== $this->on ) { - $data->on = $this->on; - } - - return $data; - - } // toJsonObj() -} // class Join diff --git a/classes/ETL/DbEntity/Query.php b/classes/ETL/DbEntity/Query.php deleted file mode 100644 index 58e63e0c0d..0000000000 --- a/classes/ETL/DbEntity/Query.php +++ /dev/null @@ -1,860 +0,0 @@ - - * @date 2015-11-20 - * ========================================================================================== - */ - -namespace ETL\DbEntity; - -use ETL\Utilities; -use \Log; -use \stdClass; - -class Query extends aNamedEntity -{ - // The list of ETL overseer restrictions supported by this query, as parsed from the query - // definition. Queries are not required to support restrictions and if a value for a restriction - // has not been set the restriction will not be applied. The ${VALUE} macro will be replaced by - // the value provided by the Overseer. For example: - // - // "source_query": { - // "overseer_restrictions": { - // "start_date": "jf.start_date >= ${VALUE}", - // "end_date": "jf.end_date <= ${VALUE}", - // "include_only_resource_codes": "jf.resource_id IN ${VALUE}", - // "exclude_resource_codes": "jf.resource_id NOT IN ${VALUE}" - // } - protected $overseerRestrictions = array(); - - // Optional array of WHERE clauses corresponding to restrictions provided by the ETL - // Overseer. These are intentionally kept separate from other query where clauses so we can - // operate on them independently. - protected $overseerRestrictionValues = array(); - - // Records describing the fields used to populate the aggregation table - protected $records = array(); - - // A 2-element array containing the field names for start and end date/times. If present this - // query support restricting the query to a particular date range. - // protected $dateFields = null; - - // Group by fields - protected $groupBys = array(); - - // Join tables. A single table generates the FROM clause while the rest are added as JOINS - protected $joins = array(); - - // Optional array of WHERE clauses - protected $where = array(); - - // Optional array of ORDER BY fields - protected $orderBys = array(); - - // Optional defined macros - protected $macros = array(); - - // Query hints (See http://dev.mysql.com/doc/refman/5.7/en/query-cache-in-select.html) - protected $queryHint = null; - - /* ------------------------------------------------------------------------------------------ - * Construct a table object from a JSON definition file or a definition object. The definition - * must contain, at a minimum, name and columns properties. - * - * @param $config Mixed Either a filename for the JSON definition file or an object containing the - * table definition - * - * Optional 2nd and 3rd arguments: - * - * @param $variableMap An associative array specifying variables that will be substituted in the - * table DDL and the aggregation SELECT statement - * @param $macroDir The directory where macro files are found. - * - * @throw Exception If the argument is not a string or instance of stdClass - * @throw Exception If the table definition was incomplete - * ------------------------------------------------------------------------------------------ - */ - - public function __construct($config, $systemQuoteChar = null, Log $logger = null) - { - parent::__construct($systemQuoteChar, $logger); - - if ( ! is_object($config) && is_string($config) ) { - $config = $this->parseJsonFile($config, "Query Definition"); - } elseif ( ! $config instanceof stdClass) { - $msg = __CLASS__ . ": Argument is not a filename or object"; - $this->logAndThrowException($msg); - } - - // Support the query config directly or assigned to a "source_query" key - - if ( isset($config->source_query) ) { - $config = $config->source_query; - } - - // Check for required properties - - $requiredKeys = array("records", "joins"); - $this->verifyRequiredConfigKeys($requiredKeys, $config); - - $this->initialize($config); - - } // __construct() - - /* ------------------------------------------------------------------------------------------ - * Verify the table. This includes ensuring any index colums match column names. - * - * @param $destinationTable The table that data from this query will be placed into. - - * @return true on success - * @throws Exception If there are errors during validation - * ------------------------------------------------------------------------------------------ - */ - - public function verify(Table $destinationTable) - { - $columnNames = $destinationTable->getColumnNames(); - $missingColumnNames = array_diff(array_keys($this->records), $columnNames); - - if ( 0 != count($missingColumnNames) ) { - $msg = "Columns in records not found in table: " . implode(", ", $missingColumnNames); - $this->logAndThrowException($msg); - } - - return true; - - } // verify() - - /* ------------------------------------------------------------------------------------------ - * Initialize internal data structures. - * - * @throws Exception if any query data was not - * int the correct format. - * ------------------------------------------------------------------------------------------ - */ - - public function initialize(stdClass $config, $force = false) - { - if ( $this->initialized && ! $force ) { - return true; - } - - // Check for required properties (records and join) - - $this->initialized = false; - $errorMsg = array(); - - if ( ! isset($config->records) ) { - $errorMsg[] = "records property not found"; - } elseif ( ! is_object($config->records) ) { - $errorMsg[] = "records property must be an object"; - } - - if ( ! isset($config->joins) ) { - $errorMsg[] = "joins property not found"; - } elseif ( ! is_array($config->joins) ) { - $errorMsg[] = "joins property must be an array"; - } elseif ( 0 == count($config->joins) ) { - $errorMsg[] = "joins property must include as least one element"; - } - - if ( isset($config->groupby) ) { - if ( ! is_array($config->groupby) ) { - $errorMsg[] = "groupby property must be an array"; - } elseif ( 0 == count($config->groupby) ) { - $errorMsg[] = "groupby property must include as least one element"; - } - } - - if ( isset($config->orderby) ) { - if ( ! is_array($config->orderby) ) { - $errorMsg[] = "orderby property must be an array"; - } elseif ( 0 == count($config->orderby) ) { - $errorMsg[] = "orderby property must include as least one element"; - } - } - - if ( isset($config->where) && ! is_array($config->where) ) { - $errorMsg[] = "where property must be an array"; - } - - if ( isset($config->macros) && ! is_array($config->macros) ) { - $errorMsg[] = "macros property must be an array"; - } - - if ( isset($config->query_hint) && ! is_string($config->query_hint) ) { - $msg = "Query hints must be a string"; - $this->logAndThrowException($msg); - } - - if ( isset($config->overseer_restrictions) && ! is_object($config->overseer_restrictions) ) { - $msg = "ETL overseer restrictions must be an object"; - $this->logger->logAndThrowException($msg); - } - - if ( 0 != count($errorMsg) ) { - $msg = "Error in query definition (" . implode(", ", $errorMsg) . ")"; - $this->logAndThrowException($msg); - } - - // Set records. Each formula must match an existing column. - - foreach ( $config->records as $column => $formula ) { - $this->addRecord($column, $formula); - } - - // Set joins. A single join is required but more may be included - - foreach ( $config->joins as $definition ) { - $this->addJoin($definition); - } - - if ( isset($config->groupby) ) { - foreach ( $config->groupby as $groupby ) { - $this->addGroupBy($groupby); - } - } - - // Set optional where clauses and macros - - if ( isset($config->where) ) { - foreach ( $config->where as $where ) { - $this->addWhere($where); - } - } - - if ( isset($config->orderby) ) { - foreach ( $config->orderby as $orderby ) { - $this->addOrderBy($orderby); - } - } - - if ( isset($config->macros) ) { - foreach ( $config->macros as $macro ) { - $this->addMacro($macro); - } - } - - if ( isset($config->query_hint) ) { - $this->setHint($config->query_hint); - } - - if ( isset($config->overseer_restrictions) ) { - foreach ( $config->overseer_restrictions as $restriction => $template ) { - $this->addOverseerRestriction($restriction, $template); - } - } - - $this->initialized = true; - - return true; - - } // initialize() - - /* ------------------------------------------------------------------------------------------ - * Add a record to this query. Records map column names to values in the SELECT statement. - * - * @param $columnName The column that the formula will be associated with. - * @param $formula The formula associated with the column. - * - * @return This object to support method chaining. - * - * @throw Exception If the column name does not exist. - * @throw Exception If the formula is empty. - * @throw Exception If the column name has already been specified. - * ------------------------------------------------------------------------------------------ - */ - - public function addRecord($columnName, $formula) - { - // Note in PHP "" and "0" are both equal to 0 due to conversion comparing strings to integers. - if ( null === $formula || "" === $formula ) { - $msg = "Empty formula for column '$columnName' '$formula'"; - $this->logAndThrowException($msg); - } elseif ( array_key_exists($columnName, $this->records) ) { - $msg = "Column '$columnName' already has a formula specified"; - $this->logAndThrowException($msg); - } - - $this->records[$columnName] = $formula; - - return $this; - - } // addRecord() - - /* ------------------------------------------------------------------------------------------ - * Get the list of records. - * - * @return An associative array where the keys are column names and the values are records - * for those columns. - * ------------------------------------------------------------------------------------------ - */ - - public function getRecords() - { - return $this->records; - } // getRecords() - - /* ------------------------------------------------------------------------------------------ - * Get a formula for the specified column. - * - * @param $columnName The column to retrieve. - * - * @return The formula for the specified column, or false if none exists. - * ------------------------------------------------------------------------------------------ - */ - - public function getRecord($columnName) - { - return ( array_key_exists($columnName, $this->records) ? $this->records[$columnName] : false ); - } // getRecord() - - /* ------------------------------------------------------------------------------------------ - * Remove all records from this query. - * - * @return This object to support method chaining. - * ------------------------------------------------------------------------------------------ - */ - - public function deleteRecords() - { - $this->records = array(); - return $this; - } // deleteRecords() - - /* ------------------------------------------------------------------------------------------ - * Remove a column record if it exists and return the formula. - * - * @param $columnName The column to remove. - * - * @return The formula for the specified column, or false if none exists. - * ------------------------------------------------------------------------------------------ - */ - - public function removeRecord($columnName) - { - $record = $this->getRecord($columnName); - if ( false !== $record ) { - unset($this->records[$columnName]); - } - return $record; - } // removeRecord() - - /* ------------------------------------------------------------------------------------------ - * Add a group by clause to this query. - * - * @param $groupBy An array containing the group by column names. - * - * @return This object to support method chaining. - * ------------------------------------------------------------------------------------------ - */ - - public function addGroupBy($groupBy) - { - if ( empty($groupBy) || ! is_string($groupBy) ) { - $msg = "Cannot add an empty group by"; - $this->logAndThrowException($msg); - } - - $this->groupBys[] = $groupBy; - return $this; - } // addGroupBys() - - /* ------------------------------------------------------------------------------------------ - * Get the list of group by columns. - * - * @return An array of group by column names. - * ------------------------------------------------------------------------------------------ - */ - - public function getGroupBys() - { - return $this->groupBys; - } // getGroupBys() - - /* ------------------------------------------------------------------------------------------ - * Remove all group bys from this query. - * - * @return This object to support method chaining. - * ------------------------------------------------------------------------------------------ - */ - - public function deleteGroupBys() - { - $this->groupBys = array(); - return $this; - } // deleteGroupBys() - - /* ------------------------------------------------------------------------------------------ - * Add a order by clause to this query. - * - * @param $orderBy An array containing the group by column names. - * - * @return This object to support method chaining. - * ------------------------------------------------------------------------------------------ - */ - - public function addOrderBy($orderBy) - { - if ( empty($orderBy) || ! is_string($orderBy) ) { - $msg = "Cannot add an empty order by"; - $this->logAndThrowException($msg); - } - - $this->orderBys[] = $orderBy; - return $this; - } // addOrderBys() - - /* ------------------------------------------------------------------------------------------ - * Get the list of order by columns. - * - * @return An array of group by column names. - * ------------------------------------------------------------------------------------------ - */ - - public function getOrderBys() - { - return $this->orderBys; - } // getOrderBys() - - /* ------------------------------------------------------------------------------------------ - * Remove all order bys from this query. - * - * @return This object to support method chaining. - * ------------------------------------------------------------------------------------------ - */ - - public function deleteOrderBys() - { - $this->orderBys = array(); - return $this; - } // deleteOrderBys() - - /* ------------------------------------------------------------------------------------------ - * Add a join clause for this query. - * - * @param $definition An object containing the column definition, or an instantiated Join - * object to add - * - * @return This object to support method chaining. - * ------------------------------------------------------------------------------------------ - */ - - public function addJoin($definition) - { - $item = ( $definition instanceof Join ? $definition : new Join($definition, $this->systemQuoteChar) ); - - if ( ! ($item instanceof iTableItem) ) { - $msg = "Join does not implement interface iTableItem"; - $this->logAndThrowException($msg); - } - - $this->joins[] = $item; - - return $this; - } // setJoins() - - /* ------------------------------------------------------------------------------------------ - * Get the list of join clauses. - * - * @return An array of join clauses - * ------------------------------------------------------------------------------------------ - */ - - public function getJoins() - { - return $this->joins; - } // getJoins() - - /* ------------------------------------------------------------------------------------------ - * Remove all joins from this query. - * - * @return This object to support method chaining. - * ------------------------------------------------------------------------------------------ - */ - - public function deleteJoins() - { - $this->joins = array(); - return $this; - } // deleteJoins() - - /* ------------------------------------------------------------------------------------------ - * Add a where clause for this query, appending to any existing where clauses. - * - * @param $where An string containing a single where clause. - * - * @return This object to support method chaining. - * ------------------------------------------------------------------------------------------ - */ - - public function addWhere($where) - { - if ( empty($where) || ! is_string($where) ) { - $msg = "WHERE clause is empty or not a string '$where'"; - $this->logAndThrowException($msg); - } - - $this->where[] = $where; - return $this; - - } // addWhere() - - /* ------------------------------------------------------------------------------------------ - * Get the list of optional where clauses. - * - * @return An array of where clauses. - * ------------------------------------------------------------------------------------------ - */ - - public function getWheres() - { - return $this->where; - } // getWheres() - - /* ------------------------------------------------------------------------------------------ - * Remove all wheres from this query. - * - * @return This object to support method chaining. - * ------------------------------------------------------------------------------------------ - */ - - public function deleteWheres() - { - $this->wheres = array(); - return $this; - } // deleteWheres() - - /* ------------------------------------------------------------------------------------------ - * Add a macro for this query, appending to any existing macros. - * - * @param $macro An object containing a single macro definition - * - * @return This object to support method chaining. - * ------------------------------------------------------------------------------------------ - */ - - public function addMacro(stdClass $macro) - { - $this->macros[] = $macro; - return $this; - } // addMacro() - - /* ------------------------------------------------------------------------------------------ - * Get the list of optional macros. - * - * @return An array of macros. - * ------------------------------------------------------------------------------------------ - */ - - public function getMacros() - { - return $this->macros; - } // getMacros() - - /* ------------------------------------------------------------------------------------------ - * Remove all macros from this query. - * - * @return This object to support method chaining. - * ------------------------------------------------------------------------------------------ - */ - - public function deleteMacros() - { - $this->macros = array(); - return $this; - } // deleteMacros() - - /* ------------------------------------------------------------------------------------------ - * Set a query hint string for the optimizer - * - * @param $hint The hint string - * - * @return This object to support method chaining. - * ------------------------------------------------------------------------------------------ - */ - - public function setHint($hint) - { - $this->queryHint = $hint; - return $this; - } // setHint() - - /* ------------------------------------------------------------------------------------------ - * Get the query hints - * - * @return The query hint string - * ------------------------------------------------------------------------------------------ - */ - - public function getHint() - { - return $this->queryHint; - } // getHint() - - /* ------------------------------------------------------------------------------------------ - * Remove all hints from this query. - * - * @return This object to support method chaining. - * ------------------------------------------------------------------------------------------ - */ - - public function deleteHint() - { - $this->queryHint = null; - return $this; - } // deleteHint() - - /* ------------------------------------------------------------------------------------------ - * Add an overseer restriction template to this query based on the parsed query definition. - * Note that at this point we don't know if the restrictions are valid (i.e., supported by the - * EtlOverseer). - * - * @param $restrictions The name of the restriction - * @param $template A template for the restriction where ${VALUE} will be replaced by the value - * - * @throws Exception if the restriction or the template are invalid - * - * @return This object to support method chaining. - * ------------------------------------------------------------------------------------------ - */ - - public function addOverseerRestriction($restriction, $template) - { - if ( ! is_string($restriction) || "" == $restriction ) { - $msg = "Overseer restriction key must be a non-empty string"; - $this->logAndThrowException($msg); - } elseif ( ! is_string($template) || "" == $template ) { - $msg = "Overseer restriction template must be a non-empty string"; - $this->logAndThrowException($msg); - } - - $this->overseerRestrictions[$restriction] = $template; - return $this; - - } // addOverseerRestriction() - - /* ------------------------------------------------------------------------------------------ - * Get the list of configured overseer restrictions. - * - * @return An associative array where the keys are restriction names and the values are the - * templates for those restrictions. - * ------------------------------------------------------------------------------------------ - */ - - public function getOverseerRestrictions() - { - return $this->overseerRestrictions; - } // getOverseerRestrictions() - - /* ------------------------------------------------------------------------------------------ - * Remove all restrictions. Note that this does not remove them from the where clause if they - * have already been added. - * - * @return This object to support method chaining. - * ------------------------------------------------------------------------------------------ - */ - - public function deleteOverseerRestrictions() - { - $this->overseerRestrictions = array(); - $this->overseerRestrictionValues = array(); - return $this; - } // deleteOverseerRestrictions() - - /* ------------------------------------------------------------------------------------------ - * Add an overseer restriction value to this query. This is the template that has been processed - * by the overseer. These values are kept separate from the other where clauses. - * - * @param $restrictions The name of the restriction - * @param $value A processed overseer restriction template - * - * @throws Exception if the restriction or the value are invalid - * - * @return The value of the specified restriction, or FALSE if the name was not found. - * ------------------------------------------------------------------------------------------ - */ - - public function addOverseerRestrictionValue($restriction, $value) - { - if ( ! is_string($restriction) || "" == $restriction ) { - $msg = "Overseer restriction key must be a non-empty string"; - $this->logAndThrowException($msg); - } elseif ( ! is_string($value) || "" == $value ) { - $msg = "Overseer restriction template must be a non-empty string"; - $this->logAndThrowException($msg); - } - - $this->overseerRestrictionValues[$restriction] = $value; - return $this; - - } // addOverseerRestrictionValue() - - /* ------------------------------------------------------------------------------------------ - * Add an overseer restriction value to this query. This is the template that has been processed - * by the overseer. These values are kept separate from the other where clauses. - * - * @param $restrictions The name of the restriction - * @param $value A processed overseer restriction template - * - * @throws Exception if the restriction or the value are invalid - * - * @return This object to support method chaining. - * ------------------------------------------------------------------------------------------ - */ - - public function getOverseerRestrictionValues() - { - return $this->overseerRestrictionValues; - } // getOverseerRestrictionValues() - - /* ------------------------------------------------------------------------------------------ - * Generate a string containing the select statement described by the configuration. This string - * may contain macros that will need to be replaced before execution. - * - * @param $includeSchema true to include the schema in the item name, if appropriate. - * - * @return An array comtaining the SQL required for altering this item. - * ------------------------------------------------------------------------------------------ - */ - - public function getSelectSql($includeSchema = true) - { - if ( 0 == count($this->joins) ) { - $msg = "At least one join is required"; - $this->logAndThrowException($msg); - } - - // Use the records to generate the SELECT columns - - $columnList = array(); - $thisObj = $this; - foreach ( $this->records as $columnName => $formula ) { - if ( $this->isComment($columnName) ) { - continue; - } - - /* - * For now, do not quote field names because we may have functions in the query. -smg - * - * $formula = implode(".", array_map(function($part) use ($thisObj) { return $thisObj->quote($part); }, explode(".", $formula))); - */ - $columnList[] = "$formula AS $columnName"; - } - - // Use the first join as the main FROM table, followined by other joins. - - $joinList = array(); - $joinList[] = "FROM " . $this->joins[0]->getCreateSql($includeSchema); - - for ($i = 1; $i < count($this->joins); $i++) { - if ( null === $this->joins[$i]->getOn() ) { - $msg = "Join clause for table '" . $this->joins[$i]->getName() . "' does not provide ON condition"; - } - - // When we move to explictly marking the FROM clause this functionality may be moved - // into the Join class - - $joinType = $this->joins[$i]->getType(); - - // Handle various join types. STRAIGHT_JOIN is a mysql enhancement. - - $joinStr = "JOIN"; - - if ( "STRAIGHT" == $joinType ) { - $joinStr = "STRAIGHT_JOIN"; - } elseif (null !== $joinType) { - $joinStr = $joinType . " JOIN"; - } - - $joinList[] = $joinStr . " " . $this->joins[$i]->getCreateSql($includeSchema); - } // for ( $i = 1; $i < count($this->joins); $i++ ) - - // Construct the SELECT statement - - // Merge in where clauses along with any overseer restrictions provided - $whereConditions = array_merge($this->where, $this->overseerRestrictionValues); - - $sql = "SELECT" .( null !== $this->queryHint ? " " . $this->queryHint : "" ) . "\n" . - implode(",\n", $columnList) . "\n" . - implode("\n", $joinList) . "\n" . - ( count($whereConditions) > 0 ? "WHERE " . implode("\nAND ", $whereConditions) . "\n" : "" ) . - ( count($this->groupBys) > 0 ? "GROUP BY " . implode(", ", $this->groupBys) : "" ) . - ( count($this->orderBys) > 0 ? "ORDER BY " . implode(", ", $this->orderBys) : "" ); - - // If any macros have been defined, process those macros now. Since macros can contain variables - // themselves, we will process the variables later. - - if (count($this->macros) > 0) { - foreach ( $this->macros as $macro ) { - $sql = Utilities::processMacro($sql, $macro); - } - } - - return $sql; - - } // getSelectSql() - - /* ------------------------------------------------------------------------------------------ - * Generate an object representation of this item suitable for encoding into JSON. - * - * @param $succinct true to use a succinct representation. - * @param $includeSchema true to include the schema in the table definition - * - * @return An object representation for this item suitable for encoding into JSON. - * ------------------------------------------------------------------------------------------ - */ - - public function toJsonObj($succinct = false, $includeSchema = false) - { - $data = new stdClass; - $data->records = $this->records; - $data->joins = $this->joins; - - if ( count($this->groupBys) > 0 ) { - $data->groupbys = $this->groupBys; - } - if ( count($this->orderBys) > 0 ) { - $data->orderbys = $this->orderBys; - } - if ( count($this->where) > 0 ) { - $data->where = $this->where; - } - if ( count($this->macros) > 0 ) { - $data->macros = $this->macros; - } - - return $data; - - } // toJsonObj() - - /* ------------------------------------------------------------------------------------------ - * Generate a JSON representation of this table. - * - * @param $succinct true if a succinct representation should be returned. - * @param $includeSchema true to include the schema in the table definition - * - * @return A JSON formatted string representing the tabe. - * ------------------------------------------------------------------------------------------ - */ - - public function toJson($succinct = false, $includeSchema = false) - { - return json_encode($this->toJsonObj($succinct, $includeSchema)); - } // toJson() -} // class Query diff --git a/classes/ETL/DbEntity/Table.php b/classes/ETL/DbEntity/Table.php deleted file mode 100644 index 28e2c5ce27..0000000000 --- a/classes/ETL/DbEntity/Table.php +++ /dev/null @@ -1,944 +0,0 @@ - - * @date 2015-10-29 - * ========================================================================================== - */ - -namespace ETL\DbEntity; - -use ETL\DataEndpoint\iRdbmsEndpoint; -use \Log; -use \stdClass; - -class Table extends aNamedEntity -{ - // Optional filename to the definition file - protected $filename = null; - - // Optional table comment - protected $comment = null; - - // Optional table engine - protected $engine = null; - - // Associative array where the keys are column names and the values are Column objects - protected $columns = array(); - - // Associative array where the keys are index names and the values are Index objects - protected $indexes = array(); - - // Associative array where the keys are trigger names and the values are Trigger objects - protected $triggers = array(); - - /* ------------------------------------------------------------------------------------------ - * Construct a table object from a JSON definition file or a definition object. The definition - * must contain, at a minimum, name and columns properties. - * - * @param $config Mixed Either a filename for the JSON definition file or an object containing the - * table definition - * - * @throw Exception If the argument is not a string or instance of stdClass - * @throw Exception If the table definition was incomplete - * ------------------------------------------------------------------------------------------ - */ - - public function __construct($config, $systemQuoteChar = null, Log $logger = null) - { - parent::__construct($systemQuoteChar, $logger); - - // If an object was passed in of stdClass assume it is the table definition, otherwise assume it - // is a filename and parse it. I am intentionally not storing the config as a property so we - // don't need to carry it around if there are many of these objects. - - if ( ! is_object($config) && is_string($config) ) { - $config = $this->parseJsonFile($config, "Table Definition"); - } elseif ( ! $config instanceof stdClass) { - $msg = __CLASS__ . ": Argument is not a filename or object"; - $this->logAndThrowException($msg); - } - - // Support the table config directly or assigned to a "table_definition" key - - if ( isset($config->table_definition) ) { - $config = $config->table_definition; - } - - // Check for required properties - - $requiredKeys = array("name", "columns"); - $this->verifyRequiredConfigKeys($requiredKeys, $config); - - $this->initialize($config); - - } // __construct() - - /* ------------------------------------------------------------------------------------------ - * @see aNamedEntity::initialize() - * ------------------------------------------------------------------------------------------ - */ - - public function initialize(stdClass $config, $force = false) - { - if ( $this->initialized && ! $force ) { - return true; - } - - $this->initialized = false; - - $this->name = $config->name; - $this->schema = ( isset($config->schema) ? $config->schema : null ); - $this->comment = ( isset($config->comment) ? $config->comment : null ); - $this->engine = ( isset($config->engine) ? $config->engine : null ); - - // Set columns. The value of columns key can be an array of column arrays (numeric keys), or an - // object containing multiple column objects. Both of these are iterables. - - $columns = $config->columns; - - foreach ( $columns as $key => $definition ) { - - if ( is_object($definition) && - ! is_numeric($key) - && ! isset($definition->name) ) { - // If the index name is not provided, allow shorthand for using the index key as the name - $definition->name = $key; - } - - $this->addColumn($definition, true); - - } // foreach ( $columns as $key => $definition ) - - - // Set indexes - - if ( isset($config->indexes) ) { - - $indexes = $config->indexes; - foreach ( $indexes as $key => $definition ) { - $this->addIndex($definition); - } - - } // if ( isset($config->indexes) ) - - - // Set triggers - - if ( isset($config->triggers) ) { - $triggers = $config->triggers; - foreach ( $triggers as $key => $definition ) { - // Default to the current table name and schema of the parent table. - if ( ! isset($definition->table) ) { - $definition->table = $this->name; - } - if ( ! isset($definition->schema) ) { - $definition->schema = $this->schema; - } - $this->addTrigger($definition); - } - } - - $this->initialized = true; - - } // initialize() - - /* ------------------------------------------------------------------------------------------ - * Verify the table. This includes ensuring any index colums match column names. - * - * @return true on success - * @throws Exception If there are errors during validation - * ------------------------------------------------------------------------------------------ - */ - - public function verify() - { - // Verify index columns match table columns - - $columnNames = $this->getColumnNames(); - - foreach ( $this->getIndexes() as $index ) { - $indexColumns = $index->getColumnNames(); - $missingColumnNames = array_diff($indexColumns, $columnNames); - if ( 0 != count($missingColumnNames) ) { - $msg = "Columns in index '" . $index->getName() . "' not found in table definition: " . - implode(", ", $missingColumnNames); - $this->logAndThrowException($msg); - } - } // foreach ( $this->getIndexes() as $index ) - - } // verify() - - /* ------------------------------------------------------------------------------------------ - * Use the MySQL information schema to build a Table object from an existing table. - * - * @param $tableName The name of the table to discover - * @param $endpoint The DataEndpoint used to connect to the database (provides schema) - * @param $systemQuoteChar The system quote character for the database that we are - * interrogating. If NULL, the system quote character will be taken from the endpoint. - * @param $log The system logger - * - * @return A Table object constructed from the table definition in MySQL, or false if the table - * name was not found. - * @throws Exception If there was an error querying or constructing the table - * ------------------------------------------------------------------------------------------ - */ - - public static function discover( - $tableName, - iRdbmsEndpoint $endpoint, - $systemQuoteChar = null, - Log $logger = null - ) { - $schemaName = null; - $qualifiedTableName = null; - - $systemQuoteChar = ( null === $systemQuoteChar - ? $endpoint->getSystemQuoteChar() - : $systemQuoteChar ); - - // If a schema was specified in the table name use it, otherwise use the default schema - - if ( false === strpos($tableName, ".") ) { - $schemaName = $endpoint->getSchema(); - $qualifiedTableName = $schemaName . "." . $tableName; - } else { - $qualifiedTableName = $tableName; - list($schemaName, $tableName) = explode(".", $tableName); - } - - $params = array(':schema' => $schemaName, - ':tablename' => $tableName); - - if ( null !== $logger ) { - $logger->debug("Discover table '$qualifiedTableName'"); - } - - // Query table properties - - $sql = "SELECT -engine, table_comment as comment -FROM information_schema.tables -WHERE table_schema = :schema -AND table_name = :tablename"; - - try { - $result = $endpoint->getHandle()->query($sql, $params); - if ( count($result) > 1 ) { - $msg = "Multiple rows returned for table"; - $this->logAndThrowException($msg); - - } - - // The table did not exist, return false - - if ( 0 == count($result) ) { - return false; - } - - } catch (Exception $e) { - $msg = "Error discovering table '$qualifiedTableName': " . $e->getMessage(); - $this->logAndThrowException($msg); - } - - $row = array_shift($result); - - $definition = array('name' => $tableName, - 'schema' => $schemaName, - 'engine' => $row['engine'], - 'columns' => array(), - 'comment' => $row['comment'] ); - - $newTable = new Table((object) $definition, $systemQuoteChar, $logger); - - // Query columns. Querying for the default needs some explaining. The information schema stores - // the default as null unless one was specifically provided so we need some logic to get things - // into the shape we want. - - // SMG: We should do a better job of detecting equivalent columns. For example "int unsigned" is - // equivalent to "int(10) unsigned". - - $sql = "SELECT -column_name as name, column_type as type, is_nullable as nullable, -column_default as " . $endpoint->quoteSystemIdentifier("default") . ", -IF('' = extra, NULL, extra) as extra, -IF('' = column_comment, NULL, column_comment) as " . $endpoint->quoteSystemIdentifier("comment") . " -FROM information_schema.columns -WHERE table_schema = :schema -AND table_name = :tablename -ORDER BY ordinal_position ASC"; - - try { - $result = $endpoint->getHandle()->query($sql, $params); - if ( 0 == count($result) ) { - $msg = "No columns returned"; - $this->logAndThrowException($msg); - } - } catch (Exception $e) { - $msg = "Error discovering table '$qualifiedTableName': " . $e->getMessage(); - $this->logAndThrowException($msg); - } - - foreach ( $result as $row ) { - $newTable->addColumn((object) $row); - } - - // Query indexes. - - $sql = "SELECT -index_name as name, index_type as " . $endpoint->quoteSystemIdentifier("type") . ", (non_unique = 0) as is_unique, -GROUP_CONCAT(column_name ORDER BY seq_in_index ASC) as columns -FROM information_schema.statistics -WHERE table_schema = :schema -AND table_name = :tablename -GROUP BY index_name -ORDER BY index_name ASC"; - - try { - $result = $endpoint->getHandle()->query($sql, $params); - } catch (Exception $e) { - $msg = "Error discovering table '$qualifiedTableName': " . $e->getMessage(); - $this->logAndThrowException($msg); - } - - foreach ( $result as $row ) { - $row['columns'] = explode(",", $row['columns']); - $newTable->addIndex((object) $row); - } - - // Query triggers - - $sql = "SELECT -trigger_name as name, action_timing as time, event_manipulation as event, -event_object_schema as " . $endpoint->quoteSystemIdentifier("schema") . ", event_object_table as " . $endpoint->quoteSystemIdentifier("table") . ", definer, -action_statement as body -FROM information_schema.triggers -WHERE event_object_schema = :schema -and event_object_table = :tablename -ORDER BY trigger_name ASC"; - - try { - $result = $endpoint->getHandle()->query($sql, $params); - } catch (Exception $e) { - $msg = "Error discovering table '$qualifiedTableName': " . $e->getMessage(); - $this->logAndThrowException($msg); - } - - foreach ( $result as $row ) { - $newTable->addTrigger((object) $row); - } - - return $newTable; - - } // discover() - - /* ------------------------------------------------------------------------------------------ - * In addition to the table schema, set the schema on any triggers if they do not have one set. - * Since a table's schema is typically set after the table is constructed, entities that can - * maintain their own schema also need to be updated. - * - * @see aNamedEntity::setSchema() - * ------------------------------------------------------------------------------------------ - */ - - public function setSchema($schema) - { - parent::setSchema($schema); - - foreach ( $this->triggers as $trigger ) { - if ( null === $trigger->getSchema() ) { - $trigger->setSchema($schema); - } - } - - return $this; - - } // setSchema() - - /* ------------------------------------------------------------------------------------------ - * @return The engine for this table - * ------------------------------------------------------------------------------------------ - */ - - public function getEngine() - { - return $this->engine; - } // getEngine() - - /* ------------------------------------------------------------------------------------------ - * @return The comment for this table - * ------------------------------------------------------------------------------------------ - */ - - public function getComment() - { - return $this->comment; - } // geComment() - - /* ------------------------------------------------------------------------------------------ - * Add a column to this table. - * - * @param $definition An array or object containing the column definition, or an instantiated - * Column object to add - * @param $overwriteDuplicates true to allow overwriting of duplicate column names. If false, throw - * an exception if a duplicate column is added. - * - * @return This object to support method chaining. - * - * @throw Exception if the new item does not implement the iTableItem interface - * @throw Exception if the new item has the same name as an existing item - * ------------------------------------------------------------------------------------------ - */ - - public function addColumn($definition, $overwriteDuplicates = false) - { - $item = ( $definition instanceof Column ? $definition : new Column($definition, $this->getSystemQuoteChar()) ); - - if ( ! ($item instanceof iTableItem) ) { - $msg = "Column does not implement interface iTableItem"; - $this->logAndThrowException($msg); - } - - $name = $item->getName(); - - if ( array_key_exists($name, $this->columns) && ! $overwriteDuplicates ) { - $this->logAndThrowException( - "Cannot add duplicate column '$name'", - array('log_level' => PEAR_LOG_WARNING) - ); - } - - $this->columns[ $name ] = $item; - - return $this; - - } // addColumn() - - /* ------------------------------------------------------------------------------------------ - * Get the list of column objects. - * - * @return An array of Column objects. - * ------------------------------------------------------------------------------------------ - */ - - public function getColumns() - { - return $this->columns; - } // getColumns() - - /* ------------------------------------------------------------------------------------------ - * Get the list of column names. - * - * @return An array of column names. - * ------------------------------------------------------------------------------------------ - */ - - public function getColumnNames() - { - return array_keys($this->columns); - } // getColumnNames() - - /* ------------------------------------------------------------------------------------------ - * Get a Column object with the specified name. - * - * @param $name The column to retrieve. - * - * @return The Column object with the specified name, or false if none exists. - * ------------------------------------------------------------------------------------------ - */ - - public function getColumn($name) - { - return ( array_key_exists($name, $this->columns) ? $this->columns[$name] : false ); - } // getColumn() - - /* ------------------------------------------------------------------------------------------ - * Remove all columns from this Table. - * - * @return This object to support method chaining. - * ------------------------------------------------------------------------------------------ - */ - - public function deleteColumns() - { - $this->columns = array(); - return $this; - } // deleteColumns() - - /* ------------------------------------------------------------------------------------------ - * Add an index to this table. - * - * @param $definition An array or object containing the index definition, or an instantiated - * Index object to add - * - * @return This object to support method chaining. - * - * @throw Exception if the new item does not implement the iTableItem interface - * ------------------------------------------------------------------------------------------ - */ - - public function addIndex($definition) - { - $item = ( $definition instanceof Index ? $definition : new Index($definition, $this->getSystemQuoteChar()) ); - - if ( ! ($item instanceof iTableItem) ) { - $msg = "Index does not implement interface iTableItem"; - $this->logAndThrowException($msg); - } - - $name = $item->getName(); - - if ( array_key_exists($name, $this->indexes) ) { - $msg = "Cannot add duplicate index '$name'"; - $this->logAndThrowException($msg); - } - - $this->indexes[ $name ] = $item; - - return $this; - - } // addIndex() - - /* ------------------------------------------------------------------------------------------ - * Get the list of index objects. - * - * @return An array of Index objects. - * ------------------------------------------------------------------------------------------ - */ - - public function getIndexes() - { - return $this->indexes; - } // getIndexes() - - /* ------------------------------------------------------------------------------------------ - * Get the list of column names. - * - * @return An array of column names. - * ------------------------------------------------------------------------------------------ - */ - - public function getIndexNames() - { - return array_keys($this->indexes); - } // getIndexNames() - - /* ------------------------------------------------------------------------------------------ - * Get an Index object with the specified name. - * - * @param $name The name of the index to retrieve. - * - * @return The Index object with the specified name. - * ------------------------------------------------------------------------------------------ - */ - - public function getIndex($name) - { - return ( array_key_exists($name, $this->indexes) ? $this->indexes[$name] : false ); - } // getIndex() - - /* ------------------------------------------------------------------------------------------ - * Remove all indexes from this Table. - * - * @return This object to support method chaining. - * ------------------------------------------------------------------------------------------ - */ - - public function deleteIndexes() - { - $this->indexes = array(); - return $this; - } // deleteIndexes() - - /* ------------------------------------------------------------------------------------------ - * Add a trigger to this table. - * - * @param $definition An array or object containing the trigger definition, or an instantiated - * Trigger object to add - * - * @return This object to support method chaining. - * - * @throw Exception if the new item does not implement the iTableItem interface - * ------------------------------------------------------------------------------------------ - */ - - public function addTrigger($definition) - { - $item = ( $definition instanceof Trigger ? $definition : new Trigger($definition, $this->getSystemQuoteChar()) ); - - if ( ! ($item instanceof iTableItem) ) { - $msg = "Trigger does not implement interface iTableItem"; - $this->logAndThrowException($msg); - } - - $this->triggers[ $item->getName() ] = $item; - - return $this; - - } // addTrigger() - - /* ------------------------------------------------------------------------------------------ - * Get the list of trigger objects. - * - * @return An array of Trigger objects. - * ------------------------------------------------------------------------------------------ - */ - - public function getTriggers() - { - return $this->triggers; - } // getTriggers() - - /* ------------------------------------------------------------------------------------------ - * Get the list of trigger names. - * - * @return An array of trigger names. - * ------------------------------------------------------------------------------------------ - */ - - public function getTriggerNames() - { - return array_keys($this->triggers); - } // getTriggerNames() - - /* ------------------------------------------------------------------------------------------ - * Get a Trigger object with the specified name. - * - * @param $name The trigger to retrieve. - * - * @return The Trigger object with the specified name. - * ------------------------------------------------------------------------------------------ - */ - - public function getTrigger($name) - { - return ( array_key_exists($name, $this->triggers) ? $this->triggers[$name] : false ); - } // getTrigger() - - /* ------------------------------------------------------------------------------------------ - * Remove all triggers from this Table. - * - * @return This object to support method chaining. - * ------------------------------------------------------------------------------------------ - */ - - public function deleteTriggers() - { - $this->triggers = array(); - return $this; - } // deleteTriggers() - - /* ------------------------------------------------------------------------------------------ - * Generate an array containing all SQL statements or fragments required to create the item. Note - * that some items (such as triggers) may require multiple statements to alter them (e.g., DROP - * TRIGGER, CREATE TRIGGER). - * - * @param $includeSchema true to include the schema in the item name, if appropriate. - * - * @return An array comtaining the SQL required for creating this item. - * ------------------------------------------------------------------------------------------ - */ - - public function getCreateSql($includeSchema = true) - { - if ( 0 == count($this->columns) ) { - return false; - } - - // Note: By using the name as the key, duplicate item names will use the last definition. This - // can occur when creating aggregation tables that contain a "year" column and an - // ${AGGREGATION_UNIT} column with an aggregation unit of "year". - - $columnCreateList = array(); - foreach ( $this->columns as $name => $column ) { - $columnCreateList[$name] = $column->getCreateSql($includeSchema); - } - - $indexCreateList = array(); - foreach ( $this->indexes as $name => $index ) { - $indexCreateList[$name] = $index->getCreateSql($includeSchema); - } - - $triggerCreateList = array(); - foreach ( $this->triggers as $name => $trigger ) { - // The table schema may have been set after the table was initially created. If the trigger - // doesn't explicitly define a schema, default to the table's schema. - if ( null === $trigger->getSchema() ) { - $trigger->setSchema($this->getSchema()); - } - $triggerCreateList[$name] = $trigger->getCreateSql($includeSchema); - } - - $tableName = ( $includeSchema ? $this->getFullName() : $this->getName(true) ); - - $sqlList = array(); - $sqlList[] = "CREATE TABLE IF NOT EXISTS $tableName (\n" . - " " . implode(",\n ", $columnCreateList) . - ( 0 != count($indexCreateList) ? ",\n " . implode(",\n ", $indexCreateList) : "" ) . "\n" . - ")" . - ( null !== $this->engine ? " ENGINE = " . $this->engine : "" ) . - ( null !== $this->comment && ! empty($this->comment) ? " COMMENT = '" . addslashes($this->comment) . "'" : "" ) . - ";"; - - foreach ( $triggerCreateList as $trigger ) { - $sqlList[] = $trigger; - } - - return $sqlList; - - } // getCreateSql() - - /* ------------------------------------------------------------------------------------------ - * Generate an array containing all SQL statements or fragments required to alter the destination - * table to match this table. Note that some items (such as triggers) may require multiple - * statements to alter them (e.g., DROP TRIGGER, CREATE TRIGGER). - * - * @param $destTable A Table object containing the defintion of the table as we would like it to - * be. - * @param $includeSchema true to include the schema in the item name, if appropriate. - * - * @return An array comtaining the SQL required for altering this item. - * ------------------------------------------------------------------------------------------ - */ - - public function getAlterSql(Table $destTable, $includeSchema = true) - { - $alterList = array(); - $triggerList = array(); - - // Update names/docs to be clearer. We are migrating $this to $dest. - - // -------------------------------------------------------------------------------- - // Process columns - - $currentColNames = $this->getColumnNames(); - $destColNames = $destTable->getColumnNames(); - - // Columns to be dropped, added, changed, or renamed - $dropColNames = array_diff($currentColNames, $destColNames); - $addColNames = array_diff($destColNames, $currentColNames); - $changeColNames = array_intersect($currentColNames, $destColNames); - $renameColNames = array(); - - // When renaming a column, be smart about it or a simple rename will mean that the new column is - // added and the old column is dropped causing potential data loss. Check for any processing - // hints on new columns: If a column is to be added, and there is a "rename_from" hint that - // matches an existing column name, mark this column to be renamed instead of added and dropped. - // We can then construct the CHANGE COLUMN statement. - - foreach ( $addColNames as $index => $addName ) { - $hint = $destTable->getColumn($addName)->getHints(); - if ( null !== $hint - && isset($hint->rename_from) - && false !== ( $hintIndex = array_search($hint->rename_from, $dropColNames) ) ) - { - $renameColNames[$hint->rename_from] = $addName; - unset($addColNames[$index]); - unset($dropColNames[$hintIndex]); - } - } // foreach ( $addColNames as $addName ) - - if ( $this->engine != $destTable->getEngine() ) { - $alterList[] = "ENGINE = " . $destTable->getEngine(); - } - - if ( $this->comment != $destTable->getComment() ) { - $alterList[] = "COMMENT = '" . addslashes($destTable->getComment()) . "'"; - } - - foreach ( $addColNames as $name ) { - $alterList[] = "ADD COLUMN " . $destTable->getColumn($name)->getCreateSql($includeSchema); - } - - foreach ( $dropColNames as $name ) { - $alterList[] = "DROP COLUMN " . $this->quote($name); - } - - foreach ( $changeColNames as $name ) { - $destColumn = $destTable->getColumn($name); - // Not all properties are required so a simple object comparison isn't possible - if ( 0 == $destColumn->compare($this->getColumn($name)) ) { - continue; - } - $alterList[] = "CHANGE COLUMN " . $destColumn->getName(true) . " " . $destColumn->getAlterSql($includeSchema); - } - - foreach ( $renameColNames as $fromColumnName => $toColumnName ) { - $destColumn = $destTable->getColumn($toColumnName); - $currentColumn = $this->getColumn($fromColumnName); - // Not all properties are required so a simple object comparison isn't possible - if ( 0 == $destColumn->compare($currentColumn) ) { - continue; - } - $alterList[] = "CHANGE COLUMN " . $currentColumn->getName(true) . " " . $destColumn->getAlterSql($includeSchema); - } - - // -------------------------------------------------------------------------------- - // Processes indexes - - $currentIndexNames = $this->getIndexNames(); - $destIndexNames = $destTable->getIndexNames(); - - $dropIndexNames = array_diff($currentIndexNames, $destIndexNames); - $addIndexNames = array_diff($destIndexNames, $currentIndexNames); - $changeIndexNames = array_intersect($currentIndexNames, $destIndexNames); - - foreach ( $dropIndexNames as $name ) { - $alterList[] = "DROP INDEX " . $this->quote($name); - } - - foreach ( $addIndexNames as $name ) { - $alterList[] = "ADD " . $destTable->getIndex($name)->getCreateSql($includeSchema); - } - - // Altered indexes need to be dropped then added - foreach ( $changeIndexNames as $name ) { - $destIndex = $destTable->getIndex($name); - // Not all properties are required so a simple object comparison isn't possible - if ( 0 == $destIndex->compare($this->getIndex($name)) ) { - continue; - } - $alterList[] = "DROP INDEX " . $destIndex->getName(true); - $alterList[] = "ADD " . $destIndex->getCreateSql($includeSchema); - } - - // -------------------------------------------------------------------------------- - // Process triggers - - // The table schema may have been set after the table was initially created. If the trigger - // doesn't explicitly define a schema, default to the table's schema. - // if ( null === $trigger->getSchema() ) $trigger->setSchema($this->getSchema()); - - $currentTriggerNames = $this->getTriggerNames(); - $destTriggerNames = $destTable->getTriggerNames(); - - $dropTriggerNames = array_diff($currentTriggerNames, $destTriggerNames); - $addTriggerNames = array_diff($destTriggerNames, $currentTriggerNames); - $changeTriggerNames = array_intersect($currentTriggerNames, $destTriggerNames); - - // Drop triggers first, then alter, then create - - foreach ( $dropTriggerNames as $name ) { - $triggerList[] = "DROP TRIGGER " . - ( null !== $this->schema && $includeSchema ? $this->quote($this->schema) . "." : "" ) . - $this->quote($name) . ";"; - } - - foreach ( $changeTriggerNames as $name ) { - $destTrigger = $destTable->getTrigger($name); - if ( 0 == $destTrigger->compare($this->getTrigger($name))) { - continue; - } - - $triggerList[] = "DROP TRIGGER " . - ( null !== $this->schema && $includeSchema ? $this->quote($this->schema) . "." : "" ) . - $this->quote($name) . ";"; - $triggerList[] = $destTable->getTrigger($name)->getCreateSql($includeSchema); - } - - foreach ( $addTriggerNames as $name ) { - $triggerList[] = $destTable->getTrigger($name)->getCreateSql($includeSchema); - } - - // -------------------------------------------------------------------------------- - // Put it all together - - if ( 0 == count($alterList) && 0 == count($triggerList) ) { - return false; - } - - $tableName = ( $includeSchema ? $this->getFullName() : $this->getName() ); - - $sqlList = array(); - if ( 0 != count($alterList) ) { - $sqlList[] = "ALTER TABLE $tableName\n" . - implode(",\n", $alterList) . ";"; - } - - if ( 0 != count($triggerList) ) { - foreach ( $triggerList as $trigger ) { - $sqlList[] = $trigger; - } - } - - return $sqlList; - - } // getAlterSql() - - /* ------------------------------------------------------------------------------------------ - * Generate an object representation of this item suitable for encoding into JSON. - * - * @param $includeSchema true to include the schema in the table definition - * - * @return An object representation for this item suitable for encoding into JSON. - * ------------------------------------------------------------------------------------------ - */ - - public function toJsonObj($succinct = false, $includeSchema = false) - { - $data = new stdClass; - $data->name = $this->name; - if ( null !== $this->schema && $includeSchema ) { - $data->schema = $this->schema; - } - if ( null !== $this->engine ) { - $data->engine = $this->engine; - } - if ( null !== $this->comment && "" != $this->comment ) { - $data->comment = $this->comment; - } - - $columns = array(); - foreach ( $this->columns as $column ) { - $columns[] = $column->toJsonObj($succinct); - } - $data->columns = $columns; - - $indexes = array(); - foreach ( $this->indexes as $index ) { - $indexes[] = $index->toJsonObj($succinct); - } - $data->indexes = $indexes; - - $triggers = array(); - foreach ( $this->triggers as $trigger ) { - $triggers[] = $trigger->toJsonObj($succinct); - } - $data->triggers = $triggers; - - return $data; - - } // toJsonObj() - - /* ------------------------------------------------------------------------------------------ - * Generate a JSON representation of this table. - * - * @param $succinct true if a succinct representation should be returned. - * @param $includeSchema true to include the schema in the table definition - * - * @return A JSON formatted string representing the tabe. - * ------------------------------------------------------------------------------------------ - */ - - public function toJson($succinct = false, $includeSchema = false) - { - return json_encode($this->toJsonObj($succinct, $includeSchema)); - } // toJson() -} // class Table diff --git a/classes/ETL/DbEntity/Trigger.php b/classes/ETL/DbEntity/Trigger.php deleted file mode 100644 index 497529b91f..0000000000 --- a/classes/ETL/DbEntity/Trigger.php +++ /dev/null @@ -1,246 +0,0 @@ - - * @date 2015-10-29 - * - * @see Table - * @see iTableItem - * ========================================================================================== - */ - -namespace ETL\DbEntity; - -use \Log; -use \stdClass; - -class Trigger extends aNamedEntity implements iTableItem -{ - // The time that the trigger is fired (before, after) - private $time = null; - - // The event that the trigger is fired on (insert, update, delete) - private $event = null; - - // The table that the trigger is associated with - private $table = null; - - // The body of the trigger - private $body = null; - - // The trigger definer for ACL purposes - private $definer = null; - - /* ------------------------------------------------------------------------------------------ - * @see iTableItem::__construct() - * ------------------------------------------------------------------------------------------ - */ - - public function __construct($config, $systemQuoteChar = null, Log $logger = null) - { - parent::__construct($systemQuoteChar, $logger); - - if ( ! is_object($config) ) { - $msg = __CLASS__ . ": Argument is not an object"; - $this->logAndThrowException($msg); - } - - $requiredKeys = array("name", "time", "event", "table", "body"); - $this->verifyRequiredConfigKeys($requiredKeys, $config); - - $this->initialize($config); - - } // __construct() - - /* ------------------------------------------------------------------------------------------ - * @see iTableItem::initialize() - * ------------------------------------------------------------------------------------------ - */ - - public function initialize(stdClass $config, $force = false) - { - if ( $this->initialized && ! $force ) { - return true; - } - - foreach ( $config as $property => $value ) { - if ( $this->isComment($property) ) { - continue; - } - - if ( ! property_exists($this, $property) ) { - $msg = "Property '$property' in config is not supported"; - $this->logAndThrowException($msg); - } - - $this->$property = $value; - - } // foreach ( $config as $property => $value ) - - $this->initialized = true; - - } // initialize() - - /* ------------------------------------------------------------------------------------------ - * @return The time that the trigger will fire, null if not specified - * ------------------------------------------------------------------------------------------ - */ - - public function getTime() - { - return $this->time; - } // getTime() - - /* ------------------------------------------------------------------------------------------ - * @return The trigger event, null if not specified - * ------------------------------------------------------------------------------------------ - */ - - public function getEvent() - { - return $this->event; - } // getEvent() - - /* ------------------------------------------------------------------------------------------ - * @return The table that the trigger is associated with, null if not specified - * ------------------------------------------------------------------------------------------ - */ - - public function getTable() - { - return $this->table; - } // getTable() - - /* ------------------------------------------------------------------------------------------ - * @return The trigger body, null if not specified - * ------------------------------------------------------------------------------------------ - */ - - public function getBody() - { - return $this->body; - } // getBody() - - /* ------------------------------------------------------------------------------------------ - * @return The trigger definer, null if not specified - * ------------------------------------------------------------------------------------------ - */ - - public function getDefiner() - { - return $this->definer; - } // getDefiner() - - /* ------------------------------------------------------------------------------------------ - * @see iTableItem::compare() - * ------------------------------------------------------------------------------------------ - */ - - public function compare(iTableItem $cmp) - { - if ( ! $cmp instanceof Trigger ) { - return 1; - } - - // Schemas are optional for the trigger - - // Triggers are considered equal if all non-null properties are the same. - - if ( $this->getName() != $cmp->getName() - || $this->getTime() != $cmp->getTime() - || $this->getEvent() != $cmp->getEvent() - || $this->getTable() != $cmp->getTable() - || $this->getBody() != $cmp->getBody() ) { - return -1; - } - - // The following properties have a default set by the database. If the property is not specified - // a value will be provided when the database information schema is queried. - - if ( ( null !== $this->getDefiner() && null !== $cmp->getDefiner() ) - && $this->getDefiner() != $cmp->getDefiner() ) { - return -1; - } - - // The following properties do not have defaults set by the database and should be considered if - // one of them is set. - - if ( ( null !== $this->getSchema() || null !== $cmp->getSchema() ) - && $this->getSchema() != $cmp->getSchema() ) { - return -1; - } - - } // compare() - - /* ------------------------------------------------------------------------------------------ - * @see iTableItem::getCreateSql() - * ------------------------------------------------------------------------------------------ - */ - - public function getCreateSql($includeSchema = false) - { - // Triggers queried from MySQL contain the begin/end but the body in the JSON may or may not. - - $addBeginEnd = ( 0 !== strpos($this->body, "BEGIN") ); - $name = ( $includeSchema ? $this->getFullName() : $this->getName(true) ); - $tableName = ( null !== $this->schema && $includeSchema ? $this->quote($this->schema) . "." : "" ) . - $this->quote($this->table); - $parts = array(); - $parts[] = "CREATE"; - if ( null !== $this->definer ) { - $parts[] = "DEFINER = {$this->definer}"; - } - $parts[] = "TRIGGER $name"; - $parts[] = $this->time; - $parts[] = $this->event; - $parts[] = "ON $tableName FOR EACH ROW\n"; - if ( $addBeginEnd ) { - $parts[] = "BEGIN\n"; - } - $parts[] = $this->body; - if ( $addBeginEnd ) { - $parts[] = "\nEND"; - } - - return implode(" ", $parts); - - } // getCreateSql() - - /* ------------------------------------------------------------------------------------------ - * @see iTableItem::getAlterSql() - * ------------------------------------------------------------------------------------------ - */ - - public function getAlterSql($includeSchema = false) - { - return $this->getCreateSql($includeSchema); - } // getAlterSql() - - /* ------------------------------------------------------------------------------------------ - * @see iTableItem::toJsonObj() - * ------------------------------------------------------------------------------------------ - */ - - public function toJsonObj($succinct = false) - { - // There is no succinct definition for a trigger - - $data = new stdClass; - $data->name = $this->name; - if ( null !== $this->schema ) { - $data->schema = $this->schema; - } - $data->time = $this->time; - $data->event = $this->event; - $data->table = $this->table; - // The body may contain newlines, these should be encoded. - $data->body = $this->body; - $data->definer = $this->definer; - - return $data; - - } // toJsonObj() -} // class Trigger diff --git a/classes/ETL/DbEntity/aNamedEntity.php b/classes/ETL/DbEntity/aNamedEntity.php deleted file mode 100644 index 66f4098795..0000000000 --- a/classes/ETL/DbEntity/aNamedEntity.php +++ /dev/null @@ -1,166 +0,0 @@ - - * @date 2016-01-25 - * ========================================================================================== - */ - -namespace ETL\DbEntity; - -use ETL\aEtlObject; -use ETL\DataEndpoint; -use ETL\DataEndpoint\DataEndpointOptions; - -use \Log; -use \stdClass; - -abstract class aNamedEntity extends aEtlObject -{ - // The optional schema name - protected $schema = null; - - // Character used to quote system identifiers. Mysql uses a backtick while postgres and oracle use - // a single quote. This is set as a static variable so we can use it in static scope in - // Table::discover() - - protected $systemQuoteChar = '`'; - - // Keys starting with this character are considered comments - const COMMENT_KEY = "#"; - - /* ------------------------------------------------------------------------------------------ - * Construct a database entity object from a JSON definition file or a definition object. - * - * @param $systemQuoteChar The character used to quote database system identifiers (may be empty) - * @param $logger PEAR Log object for system logging - * ------------------------------------------------------------------------------------------ - */ - - public function __construct($systemQuoteChar = null, Log $logger = null) - { - parent::__construct($logger); - $this->setSystemQuoteChar($systemQuoteChar); - } // __construct() - - /* ------------------------------------------------------------------------------------------ - * Return the table name. - * - * @param $quote true to wrap the name in quotes to handle special characters - * - * @return The name of this table, optionally quoted with the schema - * ------------------------------------------------------------------------------------------ - */ - - public function getName($quote = false) - { - return ( $quote ? $this->quote($this->name) : $this->name ); - } // getName() - - /* ------------------------------------------------------------------------------------------ - * @param $quote TRUE if the schema and name should be quoted, defaults to TRUE. - * - * @return The fully qualified and quoted name including the schema, if one was set. - * ------------------------------------------------------------------------------------------ - */ - - public function getFullName($quote = true) - { - return ( null !== $this->schema - ? $this->getSchema($quote) . "." - : "" ) . $this->getName($quote); - } // getFullName() - - /* ------------------------------------------------------------------------------------------ - * @param $quote true to wrap the name in quotes to handle special characters - * - * @return The schema for this table or null of no schema was set - * ------------------------------------------------------------------------------------------ - */ - - public function getSchema($quote = false) - { - return ( $quote - ? $this->quote($this->schema) - : $this->schema ); - } // getSchema() - - /* ------------------------------------------------------------------------------------------ - * Set the schema for this table. This allows us to dynamically place the table into the correct - * schema based on the destination data endpoint. - * - * @param $schema The new schema (may be empty or null) - * - * @return This object to support method chaining. - * ------------------------------------------------------------------------------------------ - */ - - public function setSchema($schema) - { - if ( null !== $schema && ! is_string($schema) ) { - $msg = "Entity schema must be null or a string"; - $this->logAndThrowException($msg); - } - - $this->schema = $schema; - return $this; - - } // setSchema() - - /* ------------------------------------------------------------------------------------------ - * @return The character used to quote system identifiers - * ------------------------------------------------------------------------------------------ - */ - - public function getSystemQuoteChar() - { - return $this->systemQuoteChar; - } // getSystemQuoteChar() - - /* ------------------------------------------------------------------------------------------ - * @param $char The character used to quote system identifiers (may be empty) - * ------------------------------------------------------------------------------------------ - */ - - public function setSystemQuoteChar($char) - { - if ( null !== $char && ! is_string($char) ) { - $msg = "System quote character must be a string"; - $this->logAndThrowException($msg); - } - - $this->systemQuoteChar = $char; - return $this; - - } // setSystemQuoteChar() - - /* ------------------------------------------------------------------------------------------ - * Wrap a system identifier in quotes appropriate for the endpint. For example, MySQL uses a - * backtick (`) to quote identifiers while Oracle and Postgres using double quotes ("). - * - * @param $identifier A system identifier (schema, table, column name) - * - * @return The identifier quoted appropriately for the endpoint - * ------------------------------------------------------------------------------------------ - */ - - public function quote($identifier) - { - return $this->systemQuoteChar . $identifier . $this->systemQuoteChar; - } // quote() - - /* ------------------------------------------------------------------------------------------ - * Identify commented out keys in JSON definition/specification files. - * - * @param $key The string to examine - * - * @return TRUE if the key is considered a comment, FALSE otherwise. - * ------------------------------------------------------------------------------------------ - */ - - protected function isComment($key) - { - return ( 0 === strpos($key, self::COMMENT_KEY) ); - } // isComment() -} // abstract class aNamedEntity diff --git a/classes/ETL/DbEntity/iTableItem.php b/classes/ETL/DbEntity/iTableItem.php deleted file mode 100644 index a9f1726e5a..0000000000 --- a/classes/ETL/DbEntity/iTableItem.php +++ /dev/null @@ -1,106 +0,0 @@ - - * @date 2015-10-29 - * ========================================================================================== - */ - -namespace ETL\DbEntity; - -use \Log; - -interface iTableItem -{ - /* ------------------------------------------------------------------------------------------ - * The contructor MUST provide a configuration specification, which may be an array, object, or - * file depending on the type of table item. Additional arguments MAY be provided and will be - * handled by the individual contructors using func_get_args() - * - * @param $config An object or an array containing the item definition, or possibly a file name if - * supported by the particular item. - * @param $systemQuoteChar Character used for escaping system identifiers in queries. - * - * @throw Exception If an invalid nummber of arguments was provided - * @throw Exception If the column definition was incomplete - * ------------------------------------------------------------------------------------------ - */ - - public function __construct($config, $systemQuoteChar = null, Log $logger = null); - - /* ------------------------------------------------------------------------------------------ - * @param $quote TRUE to wrap the name in quotes to handle special characters - * - * @return The name of this item - * ------------------------------------------------------------------------------------------ - */ - - public function getName($quote = false); - - /* ------------------------------------------------------------------------------------------ - * @return The fully qualified and quoted name including the schema, if one was set. - * ------------------------------------------------------------------------------------------ - */ - - public function getFullName(); - - /* ------------------------------------------------------------------------------------------ - * @return A string representation for this item. The format of the representation is flexible. - * ------------------------------------------------------------------------------------------ - */ - - public function __toString(); - - /* ------------------------------------------------------------------------------------------ - * Compare the specified object to this one and return 0 of the objects are the same, -1 if the - * specified object is considered less than this object, or 1 of the specified object is - * considered greater than this object. Not all object support the concept of greater or less than - * in which case any non-zero value is considered different. - * - * @return 0 of the objects are the same, -1 if the specified object is considered less than this - * object, or 1 of the specified object is considered greater than this object. - * ------------------------------------------------------------------------------------------ - */ - - public function compare(iTableItem $cmp); - - /* ------------------------------------------------------------------------------------------ - * Generate an object representation of this item suitable for encoding into JSON. This is - * designed to be called recursively to build up the representation of a Table object. - * - * @param $succinct Flag indicating whether or not the succinct (array) representation should be - * returned as opposed to the object notation. - * - * @return An object representation for this item suitable for encoding into JSON. - * ------------------------------------------------------------------------------------------ - */ - - public function toJsonObj($succinct = false); - - /* ------------------------------------------------------------------------------------------ - * Generate an array containing all SQL statements or fragments required to create the item. Note - * that some items (such as triggers) may require multiple statements to alter them (e.g., DROP - * TRIGGER, CREATE TRIGGER). - * - * @param $includeSchema TRUE to include the schema in the item name, if appropriate. - * - * @return An string comtaining the SQL required for creating this item. - * ------------------------------------------------------------------------------------------ - */ - - public function getCreateSql($includeSchema = false); - - /* ------------------------------------------------------------------------------------------ - * Generate an array containing all SQL statements or fragments required to alter the item. Note - * that some items (such as triggers) may require multiple statements to alter them (e.g., DROP - * TRIGGER, CREATE TRIGGER). - * - * @param $includeSchema TRUE to include the schema in the item name, if appropriate. - * - * @return An string comtaining the SQL required for altering this item. - * ------------------------------------------------------------------------------------------ - */ - - public function getAlterSql($includeSchema = false); -} // interface iTableItem diff --git a/classes/ETL/DbModel/AggregationTable.php b/classes/ETL/DbModel/AggregationTable.php new file mode 100644 index 0000000000..39fc4a0575 --- /dev/null +++ b/classes/ETL/DbModel/AggregationTable.php @@ -0,0 +1,237 @@ + + * @date 2017-04-29 + * ========================================================================================== + */ + +namespace ETL\DbModel; + +use ETL\Utilities; +use Log; +use stdClass; + +class AggregationTable extends Table +{ + // Properties required by this class. These will be merged with other required + // properties up the call chain. See @Entity::$requiredProperties + private $localRequiredProperties = array( + 'table_prefix' + ); + + // Properties provided by this class. These will be merged with other properties up + // the call chain. See @Entity::$properties + private $localProperties = array( + // Current aggregation unit to use when generating the SQL to populate the table + 'aggregation_unit' => null, + + // Table prefix used to generate the name along with the aggregation unit + 'table_prefix' => null, + + // Query object for populating the table with data + 'query' => null + ); + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::__construct() + * ------------------------------------------------------------------------------------------ + */ + + public function __construct($config, $systemQuoteChar = null, Log $logger = null) + { + // Property merging is performed first so the values can be used in the constructor + parent::mergeProperties($this->localRequiredProperties, $this->localProperties); + parent::__construct($config, $systemQuoteChar, $logger); + } // __construct() + + /* ------------------------------------------------------------------------------------------ + * @see aNamedEntity::initialize() + * ------------------------------------------------------------------------------------------ + */ + + public function initialize(stdClass $config) + { + // Aggregation table definition files may include the query used to populate the + // aggregation table. + + if ( isset($config->source_query) ) { + $this->query = $config->source_query; + unset($config->source_query); + } + + parent::initialize($config); + + } // initialize() + + /* ------------------------------------------------------------------------------------------ + * @see Entity::filterAndVerifyValue() + * ------------------------------------------------------------------------------------------ + */ + + protected function filterAndVerifyValue($property, $value) + { + $value = parent::filterAndVerifyValue($property, $value); + + if ( null === $value ) { + return $value; + } + + switch ( $property ) { + case 'query': + if ( ! is_object($value) ) { + $this->logAndThrowException( + sprintf("%s name must be an object, '%s' given", $property, gettype($value)) + ); + } + break; + + case 'aggregation_unit': + case 'table_prefix': + if ( ! is_string($value) ) { + $this->logAndThrowException( + sprintf("%s name must be a string, '%s' given", $property, gettype($value)) + ); + } + break; + + default: + break; + } // switch ( $property ) + + return $value; + + } // filterAndVerifyValue() + + /* ------------------------------------------------------------------------------------------ + * Aggregation tables support variables (e.g., ${AGGREGATION_UNIT) as defined by the aggregation + * machinery) in the column, index, and trigger definitions. These variables must be replaced with + * values prior to executing DDL statements and prior to comparing tables for generation of ALTER + * TABLE statements (e.g., prior to calling aDatabaseDestinationAction::manateTable(). + * + * Create a copy of the current table and perform variable substitution on all column, index, and + * trigger definitions. + * + * @return A copy (clone) of this table with variable substiution performed on the column, index, + * and trigger definition fields. + * ------------------------------------------------------------------------------------------ + */ + + public function copyAndApplyVariables(array $variableMap) + { + // Save the JSON representation for columns, indexes, triggers + + $columnJson = array(); + $indexJson = array(); + $triggerJson = array(); + + foreach ( $this->columns as $column ) { + $columnJson[] = $column->toStdClass(); + } + foreach ( $this->indexes as $index ) { + $indexJson[] = $index->toStdClass(); + } + foreach ( $this->triggers as $trigger ) { + $triggerJson[] = $trigger->toStdClass(); + } + + // Clone this object and clear the existing columns, indexes, and triggers. Add them using the + // saved definitions after substitutions have been performed. + + $newTable = clone $this; + // We can't set columns to null because it is a required column + $newTable->columns = array(); + $newTable->indexes = array(); + $newTable->triggers = array(); + + foreach ( $columnJson as $def ) { + if ( null !== $variableMap ) { + foreach ( $def as $key => &$value ) { + $value = Utilities::substituteVariables($value, $variableMap); + } + unset($value); // Sever the reference with the last element + } + + // Add the column, allowing duplicate column names to overwrite previous values. Without + // overwrite turned on, the yearly aggregation tables with throw an exception and log a + // warning for the year_id column. + + $newTable->addColumn($def, true); + + } + + foreach ( $indexJson as $def ) { + if ( null !== $variableMap ) { + foreach ( $def as $key => &$value ) { + $value = Utilities::substituteVariables($value, $variableMap); + } + unset($value); // Sever the reference with the last element + } + $newTable->addIndex($def); + } + + foreach ( $triggerJson as $def ) { + if ( null !== $variableMap ) { + foreach ( $def as $key => &$value ) { + $value = Utilities::substituteVariables($value, $variableMap); + } + unset($value); // Sever the reference with the last element + } + $newTable->addTrigger($def); + } + + return $newTable; + + } // copyAndApplyVariables() + + /* ------------------------------------------------------------------------------------------ + * @see Entity::__set() + * ------------------------------------------------------------------------------------------ + */ + + public function __set($property, $value) + { + // If we are not setting a property that is a special case, just call the main setter + $specialCaseProperties = array('aggregation_unit', 'query'); + + if ( ! in_array($property, $specialCaseProperties) ) { + parent::__set($property, $value); + return; + } + + // Verify values prior to doing anything with them + + $value = $this->filterAndVerifyValue($property, $value); + + // Handle special cases. + + switch ($property) { + case 'aggregation_unit': + parent::__set($property, $value); + // When setting the aggregation unit, update the table name to include it + $this->name = $this->table_prefix . $value; + break; + + case 'query': + $this->properties[$property] = null; + if ( null !== $value ) { + $query = ( is_object($value) && $value instanceof Query + ? $value + : new Query($value, $this->systemQuoteChar, $this->logger) ); + $this->properties[$property] = $query; + } + break; + + default: + break; + } // switch($property) + + } // __set() +} // class AggregationTable diff --git a/classes/ETL/DbEntity/Column.php b/classes/ETL/DbModel/Column.php similarity index 55% rename from classes/ETL/DbEntity/Column.php rename to classes/ETL/DbModel/Column.php index f692f40ba8..cf54f40a13 100644 --- a/classes/ETL/DbEntity/Column.php +++ b/classes/ETL/DbModel/Column.php @@ -1,123 +1,99 @@ - * @date 2015-10-29 + * @date 2017-04-28 * * @see Table - * @see iTableItem + * @see iEntity * ========================================================================================== */ -namespace ETL\DbEntity; +namespace ETL\DbModel; -use \Log; -use \stdClass; +use Log; -class Column extends aNamedEntity implements iTableItem +class Column extends NamedEntity implements iEntity { - // Column type (free-form string) - private $type = null; - - // true if the column is nullable - private $nullable = null; - - // Column default - private $default = null; - - // Column extra (see http://dev.mysql.com/doc/refman/5.7/en/create-table.html) - private $extra = null; - - // Column comment - private $comment = null; - - // Column hints objecct - private $hints = null; + // Properties required by this class. These will be merged with other required + // properties up the call chain. See @Entity::$requiredProperties + private $localRequiredProperties = array( + 'type' + ); + + // Properties provided by this class. These will be merged with other properties up + // the call chain. See @Entity::$properties + private $localProperties = array( + // Column type (free-form string) + 'type' => null, + // TRUE if the column is nullable + 'nullable' => null, + // Column default + 'default' => null, + // Column extra (see http://dev.mysql.com/doc/refman/5.7/en/create-table.html) + 'extra' => null, + // The trigger definer for ACL purposes + 'comment' => null, + // Column hints object + 'hints' => null, + // The trigger definer for ACL purposes + 'rename' => null + ); /* ------------------------------------------------------------------------------------------ - * @see iTableItem::__construct() + * @see iEntity::__construct() * ------------------------------------------------------------------------------------------ */ public function __construct($config, $systemQuoteChar = null, Log $logger = null) { - parent::__construct($systemQuoteChar, $logger); - - if ( ! is_object($config) ) { - $msg = __CLASS__ . ": Column definition must be an object"; - $this->logAndThrowException($msg); - } - - $requiredKeys = array("name", "type"); - $this->verifyRequiredConfigKeys($requiredKeys, $config); - - $this->initialize($config); - + // Property merging is performed first so the values can be used in the constructor + parent::mergeProperties($this->localRequiredProperties, $this->localProperties); + parent::__construct($config, $systemQuoteChar, $logger); } // __construct() /* ------------------------------------------------------------------------------------------ - * @see aNamedEntity::initialize() + * @see Entity::filterAndVerifyValue() * ------------------------------------------------------------------------------------------ */ - public function initialize(stdClass $config, $force = false) + protected function filterAndVerifyValue($property, $value) { - if ( $this->initialized && ! $force ) { - return true; - } + $value = parent::filterAndVerifyValue($property, $value); - foreach ( $config as $property => $value ) { - if ( $this->isComment($property) ) { - continue; - } - - if ( ! property_exists($this, $property) ) { - $msg = "Property '$property' in config is not supported"; - $this->logAndThrowException($msg); - } - - $this->$property = $this->filterValue($property, $value); - } // foreach ( $config as $property => $value ) - - $this->initialized = true; - - } // initialize() - - /* ------------------------------------------------------------------------------------------ - * Filter values based on the property. Some properties are true/false but may be specified as - * true, null, or YES depending on the input source. Other properties may be empty strings when - * discovered from the database which should be treated as null for our purposes - * - * @param $property The property we are filtering - * @param $value The value of the property as presented from the source (array, object, database) - * - * @return The filtered value - * ------------------------------------------------------------------------------------------ - */ + if ( null === $value ) { + return $value; + } - private function filterValue($property, $value) - { switch ( $property ) { case 'nullable': - // The config files use "NULL" and "NOT NULL" but the MySQL information schema uses "YES" and - // "NO" - $tmp = strtolower($value); - $tmp = ( "null" == $tmp ? true : $tmp ); + $origValue = $value; $value = \xd_utilities\filter_var($value, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); + if ( null === $value ) { + $this->logAndThrowException( + sprintf("%s must be a boolean, '%s' given", $property, gettype($origValue)) + ); + } break; case 'extra': case 'comment': - // If these values are empty they are considered null - $value = ( empty($value) ? null : $value ); + if ( ! is_string($value) ) { + $this->logAndThrowException( + sprintf("%s name must be a string, '%s' given", $property, gettype($value)) + ); + } break; case 'type': - // MySQL stores the column type in lowercase - $value = strtolower($value); + if ( ! is_string($value) ) { + $this->logAndThrowException( + sprintf("%s name must be a string, '%s' given", $property, gettype($value)) + ); + } break; default: @@ -125,75 +101,15 @@ private function filterValue($property, $value) } // switch ( $property ) return $value; - } // filterValue() - - /* ------------------------------------------------------------------------------------------ - * @return The type of this column - * ------------------------------------------------------------------------------------------ - */ - - public function getType() - { - return $this->type; - } // getType() - - /* ------------------------------------------------------------------------------------------ - * @return true if this column is nullable, false if it is not, or null if the property is not - * set. - * ------------------------------------------------------------------------------------------ - */ - - public function isNullable() - { - return $this->nullable; - } // isNullable() - - /* ------------------------------------------------------------------------------------------ - * @return The default value for this column, or null if the property is not set. - * ------------------------------------------------------------------------------------------ - */ - - public function getDefault() - { - return $this->default; - } // getDefault() - /* ------------------------------------------------------------------------------------------ - * @return The "extra" value for this column, or null if the property is not set. - * ------------------------------------------------------------------------------------------ - */ - - public function getExtra() - { - return $this->extra; - } // getExtra() - - /* ------------------------------------------------------------------------------------------ - * @return The comment for this column, or null if the property is not set. - * ------------------------------------------------------------------------------------------ - */ - - public function getComment() - { - return $this->comment; - } // getComment() + } // filterAndVerifyValue() /* ------------------------------------------------------------------------------------------ - * @return The hints for this column, or null if the property is not set. + * @see iEntity::compare() * ------------------------------------------------------------------------------------------ */ - public function getHints() - { - return $this->hints; - } // getHints() - - /* ------------------------------------------------------------------------------------------ - * @see iTableItem::compare() - * ------------------------------------------------------------------------------------------ - */ - - public function compare(iTableItem $cmp) + public function compare(iEntity $cmp) { if ( ! $cmp instanceof Column ) { return 1; @@ -206,12 +122,8 @@ public function compare(iTableItem $cmp) // Note that the "enum" type will be handled in a special case below so only match types here // that are different and are not both enumerated. - if ( $this->getName() != $cmp->getName() - || - ( $this->getType() != $cmp->getType() - && ! (0 === strpos($this->getType(), 'enum') - && 0 === strpos($cmp->getType(), 'enum')) - ) + if ( $this->name != $cmp->name + || ( $this->type != $cmp->type && ! (0 === strpos($this->type, 'enum') && 0 === strpos($cmp->type, 'enum')) ) ) { return -1; } @@ -219,15 +131,15 @@ public function compare(iTableItem $cmp) // Timestamp fields have special handling for default and extra fields. // See https://dev.mysql.com/doc/refman/5.5/en/timestamp-initialization.html - if ( "timestamp" == $this->getType() ) { + if ( "timestamp" == $this->type ) { // In the mode where a config file is compared to an existing database, the source is // considered the configuration while the destination is the database. - $srcDefault = $this->getDefault(); - $destDefault = $cmp->getDefault(); - $srcExtra = $this->getExtra(); - $destExtra = $cmp->getExtra(); + $srcDefault = $this->default; + $destDefault = $cmp->default; + $srcExtra = $this->extra; + $destExtra = $cmp->extra; // MySQL considers the following equivalent to CURRENT_TIMESTAMP and will convert them // automatically. Map them now so we don't get into an endless ALTER TABLE loop. @@ -309,26 +221,24 @@ public function compare(iTableItem $cmp) } else { // The following properties do not have defaults set by the database and should be considered if // one of them is set. - if ( ( null !== $this->getDefault() || null !== $cmp->getDefault() ) - && $this->getDefault() != $cmp->getDefault() ) { + if ( ( null !== $this->default || null !== $cmp->default ) && $this->default != $cmp->default ) { return -1; } - if ( ( null !== $this->getExtra() || null !== $cmp->getExtra() ) - && $this->getExtra() != $cmp->getExtra() ) { + if ( ( null !== $this->extra || null !== $cmp->extra ) && $this->extra != $cmp->extra ) { return -1; } - } // else ( "timestamp" == $this->getType() ) + } // else ( "timestamp" == $this->type ) // The enum type may be formatted by the database to add spaces between parameter // values. Normalize the values before comparing. - if ( 0 === ($myStartPos = strpos($this->getType(), 'enum')) - && 0 === ($cmpStartPos = strpos($cmp->getType(), 'enum')) ) { + if ( 0 === ($myStartPos = strpos($this->type, 'enum')) + && 0 === ($cmpStartPos = strpos($cmp->type, 'enum')) ) { // Extract the enum value list and normalize it to include no spaces between values - $myType = substr($this->getType(), 4); + $myType = substr($this->type, 4); $myType = implode(',', preg_split('/\s*,\s*/', trim($myType, "() \t\n\r\0\x0B"))); - $cmpType = substr($cmp->getType(), 4); + $cmpType = substr($cmp->type, 4); $cmpType = implode(',', preg_split('/\s*,\s*/', trim($cmpType, "() \t\n\r\0\x0B"))); if ( $myType != $cmpType ) { return -1; @@ -338,16 +248,14 @@ public function compare(iTableItem $cmp) // The following properties have a default set by the database. If the property is not specified // a value will be provided when the database information schema is queried. - if ( ( null !== $this->isNullable() && null !== $cmp->isNullable() ) - && $this->isNullable() != $cmp->isNullable() ) { + if ( ( null !== $this->nullable && null !== $cmp->nullable ) && $this->nullable != $cmp->nullable ) { return -1; } // The following properties do not have defaults set by the database and should be considered if // one of them is set. - if ( ( null !== $this->getComment() || null !== $cmp->getComment() ) - && $this->getComment() != $cmp->getComment() ) { + if ( ( null !== $this->comment || null !== $cmp->comment ) && $this->comment != $cmp->comment ) { return -1; } @@ -356,11 +264,11 @@ public function compare(iTableItem $cmp) } // compare() /* ------------------------------------------------------------------------------------------ - * @see iTableItem::getCreateSql() + * @see iEntity::getSql() * ------------------------------------------------------------------------------------------ */ - public function getCreateSql($includeSchema = false) + public function getSql($includeSchema = false) { // Name and type are required. null values are treated as not provided/specified. @@ -417,61 +325,5 @@ public function getCreateSql($includeSchema = false) return implode(" ", $parts); - } // getCreateSql() - - /* ------------------------------------------------------------------------------------------ - * @see iTableItem::getAlterSql() - * ------------------------------------------------------------------------------------------ - */ - - public function getAlterSql($includeSchema = false) - { - return $this->getCreateSql($includeSchema); - } // getAlterSql() - - /* ------------------------------------------------------------------------------------------ - * @see iTableItem::toJsonObj() - * ------------------------------------------------------------------------------------------ - */ - - public function toJsonObj($succinct = false) - { - if ( $succinct ) { - - $data = array($this->name, $this->type); - if ( null !== $this->nullable ) { - $data[] = ( $this->nullable ? "null" : "not null" ); - } - if ( null !== $this->default ) { - $data[] = $this->default; - } - if ( null !== $this->extra) { - $data[] = $this->extra; - } - if ( null !== $this->comment) { - $data[] = $this->comment; - } - - } else { - - $data = new stdClass; - $data->name = $this->name; - $data->type = $this->type; - if ( null !== $this->nullable ) { - $data->nullable = $this->nullable; - } - if ( null !== $this->default ) { - $data->default = $this->default; - } - if ( null !== $this->extra) { - $data->extra = $this->extra; - } - if ( null !== $this->comment) { - $data->comment = $this->comment; - } - } - - return $data; - - } // toJsonObj() + } // getSql() } // class Column diff --git a/classes/ETL/DbModel/Entity.php b/classes/ETL/DbModel/Entity.php new file mode 100644 index 0000000000..fb71a7ec45 --- /dev/null +++ b/classes/ETL/DbModel/Entity.php @@ -0,0 +1,503 @@ +initialize($config) + * $this->verifyRequiredProperties() + * foreach ( $config as $name => $value ) { + * $this->__set($name, $value) + * $this->namme = $this->filterAndVerifyValue($value); + * } + * + * The general steps to follow when extending this class are: + * + * 1. Define 2 private arrays, $localProperties and $localRequiredProperties, that specify + * the additional properties that a child object implements and requires. + * 2. In the child constructor, call mergeProperties() to merge local properties into a + * master list prior to calling the parent constructor. This will allow all locally + * defined properties to be merged up the chain of extended objects and also preserve + * the default values for the properties. + * 3. Override filterAndVerifyValue() in the child class to handle properties defined by + * that class. + * 4. In some cases initialize() may need to be overriden. For example, to set an index + * name before calling parent::initialize() to set the values. + * 5. Entity::__set() provides a mechanism to easily set values. If the child class + * implements any non-scalar data members the __set() will need to be overriden to + * handle that data. For example, a Query defines a "joins" property that is an array + * of Join objects. Query::__set() should handle the creation of the Join objects and + * call parent::__set() for the scalar data. It may be useful to define additional + * methods such as addRecord() or addColumn() to append data to a property rather than + * overwriting it. + * 6. Entity::__get() provides accessors to the data but it may be useful to define + * additional methods that allow you to address individual or named elements of + * associative arrays, for example. + * + * @author Steve Gallo + * @date 2017-04-27 + * ========================================================================================== + */ + +namespace ETL\DbModel; + +use Log; +use stdClass; +use ETL\Loggable; +use ETL\DataEndpoint; +use ETL\DataEndpoint\DataEndpointOptions; + +class Entity extends Loggable +{ + // The list of required properties for this model. If extending classes define their + // own required properties they should merge them in the constructor by calling + // Entity::mergeProperties($localRequiredProperties). + protected $requiredProperties = array(); + + // Associative array of valid (property, initial value) pairs. Properties may be set + // via __set(), initialize() or a custom method. . If extending classes define their + // own properties they should merge them in the constructor by calling + // Entity::mergeProperties($localProperties). + protected $properties = array(); + + // Associative array of valid (property, initial value) pairs. This contains the + // original default values. + private $defaultPropertyValues = array(); + + // Character used to quote system identifiers. Mysql uses a backtick while postgres + // and oracle use a single quote. This is set as a static variable so we can use it in + // static scope in Table::discover() + protected $systemQuoteChar = '`'; + + /* ------------------------------------------------------------------------------------------ + * The default type of the $config is a stdClass object. If a child class wishes to + * implement a different type (such as a file) it is free to do so and then pass NULL + * to this constructor. + * + * @see iEntity::__construct() + * ------------------------------------------------------------------------------------------ + */ + + public function __construct($config, $systemQuoteChar = null, Log $logger = null) + { + parent::__construct($logger); + $this->setSystemQuoteChar($systemQuoteChar); + + // The configuration can be NULL (nothing is initialized), a string assumed to be + // a path to a JSON file (the file is parsed), or a stdClass containing an + // configuration object. + + if ( null !== $config ) { + + if ( is_string($config) ) { + $config = $this->parseJsonFile($config, "Table Definition"); + // Support the table config directly or assigned to a "table_definition" key + if ( isset($config->table_definition) ) { + $config = $config->table_definition; + } + } + + if ( is_object($config) ) { + if ( ! $config instanceof stdClass ) { + $this->logAndThrowException( + sprintf("Config must be a stdClass object, '%s' given", get_class($config)) + ); + } + $this->initialize($config); + } + } + + } // __construct() + + /* ------------------------------------------------------------------------------------------ + * Merge properties and required properties local to a child class into the master + * property list. Also keep a list of the default values for all properties. This + * should be called in the child class constructor to merge any local properties + * defined. + * + * @param array $localRequiredProperties Required properties from a child class + * @param array $localProperties Locally defined properties from a child class + * + * @return This object + * ------------------------------------------------------------------------------------------ + */ + + protected function mergeProperties(array $localRequiredProperties, array $localProperties) + { + $this->requiredProperties = array_merge($this->requiredProperties, $localRequiredProperties); + $this->properties = array_merge($this->properties, $localProperties); + $this->defaultPropertyValues = array_merge($this->properties, $localProperties); + return $this; + } // mergeProperties() + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::initialize() + * + * 1. Prior to initializing the properties, verify that all required properties are + * present in the $config object. + * 2. The __set() method is used to actually set the value of each property, verified + * and filtered values, and ensure only supported properties will be set. This is + * enforced by __set(). + * ------------------------------------------------------------------------------------------ + */ + + public function initialize(stdClass $config) + { + $this->verifyRequiredProperties($config); + + foreach ( $config as $property => $value ) { + // The actual assignment will be handled by __set() including a warning for + // attempting to set an unsupported property + $this->$property = $value; + } + + return $this; + + } // initialize() + + /* ------------------------------------------------------------------------------------------ + * Verify that all of the required properties are present in the configuration object. + * + * @param stdClass $config The configuration object that we are verifying + * + * @return TRUE if the configuration object was successfully verified, FALSE otherwise. + * + * @throw Exception if a the configuration object is missing a property + * ------------------------------------------------------------------------------------------ + */ + + protected function verifyRequiredProperties(stdClass $config) + { + $missing = array(); + + foreach ( $this->requiredProperties as $required ) { + if ( ! isset($config->$required) ) { + $missing[] = $required; + } + } + + if ( 0 != count($missing) ) { + $this->logAndThrowException( + sprintf('Config missing required properties (%s)', implode(", ", $missing)) + ); + } + + return ( 0 == count($missing) ); + + } // verifyRequiredProperties() + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::verify() + * ------------------------------------------------------------------------------------------ + */ + + public function verify() + { + // Do nothing. Child classes should override this method as needed. + return true; + } // verify() + + /* ------------------------------------------------------------------------------------------ + * Filter and verify values when they are set based on the property. Some properties + * are true/false but may be specified as true, null, or YES depending on the input + * source. Other properties may be empty strings when discovered from the database + * which should be treated as null for our purposes. This is an empty method that + * should be overriden by child classes to filter their locally defined values as + * needed. + * + * In addition, errors should be thrown if values cannot be filtered to the correct + * type. + * + * @param $property The property we are filtering + * @param $value The value of the property as presented from the source + * + * @return The filtered value + * ------------------------------------------------------------------------------------------ + */ + + protected function filterAndVerifyValue($property, $value) + { + // Required properties cannot be NULL + + if ( in_array($property, $this->requiredProperties) && null === $value ) { + $this->logAndThrowException(sprintf("Required property %s cannot be null", $property)); + } elseif ( null === $value ) { + return $this->defaultPropertyValues[$property]; + } + + return $value; + } // filterAndVerifyValue() + + /* ------------------------------------------------------------------------------------------ + * @return The character used to quote system identifiers + * ------------------------------------------------------------------------------------------ + */ + + public function getSystemQuoteChar() + { + return $this->systemQuoteChar; + } // getSystemQuoteChar() + + /* ------------------------------------------------------------------------------------------ + * @param $char The character used to quote system identifiers (may be empty) + * ------------------------------------------------------------------------------------------ + */ + + public function setSystemQuoteChar($char) + { + if ( null !== $char && ! is_string($char) ) { + $this->logAndThrowException("System quote character must be a string"); + } + + $this->systemQuoteChar = $char; + + return $this; + + } // setSystemQuoteChar() + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::quote() + * ------------------------------------------------------------------------------------------ + */ + + public function quote($identifier) + { + // Don't quote the identifier if it's already been quoted + + if (0 === strpos($identifier, $this->systemQuoteChar) + && (strlen($identifier) - 1) === strrpos($identifier, $this->systemQuoteChar) ) { + return $identifier; + } else { + return $this->systemQuoteChar . $identifier . $this->systemQuoteChar; + } + + } // quote() + + /* ------------------------------------------------------------------------------------------ + * Parse a JSON table configuration file. + * + * @param $filename The file containing the table configuration + * @param $name Optional name for the file. Useful for error reporting. + * + * @return This object to support method chaining. + * + * @throw Exception If the file is does not exist or is not readable + * @throw Exception If there is an error parsing the file + * ------------------------------------------------------------------------------------------ + */ + + protected function parseJsonFile($filename, $name = null) + { + $name = ( null === $name ? "JSON file" : $name ); + $opt = new DataEndpointOptions(array('name' => $name, + 'path' => $filename, + 'type' => "jsonfile")); + $jsonFile = DataEndpoint::factory($opt, $this->logger); + return $jsonFile->parse(); + } // parseJsonFile() + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::toStdClass() + * + * Translate this entity into a simple stdClass object. An attempt will be made to + * replicate the original configuration and any changes as closely as possible, but + * child classes may need to extend this class to properly handle cComposite objects + * or different representations of the data. + * ------------------------------------------------------------------------------------------ + */ + + public function toStdClass() + { + $data = new stdClass; + + // Should we implement a recursive method for handling arrays and objects here so + // we don't need to define this method in composite children like Query and Table? + + foreach ( $this->properties as $property => $value) { + $data->$property = $this->_toStdClass($value); + } + + return $data; + + } // toStdClass() + + /* ------------------------------------------------------------------------------------------ + * Perform a recursive conversion of a value to a stdClass. Any complex values (i.e., + * objects) should implement the iEntity interface so their toStdClass() methods can + * be used to perform the conversion. If they do not, get_object_vars() will be used + * instead. + * + * This method tries to preserve the data as it is represented in this and any child + * classes, but a child class may need to extend this method to properly translate its + * local data representation into a stdClass object capable of being used as a + * configuration object for that class. + * + * @see iEntity::toStdClass() + * ------------------------------------------------------------------------------------------ + */ + + // @codingStandardsIgnoreLine + private function _toStdClass($value) + { + // Note that an empty stdClass will be treated as an array and must be overriden + // in a child class. + + if ( is_array($value) && count($value) > 0 ) { + + // Attempt to maintain objects or arrays. If all of the array keys are numeric + // assume it is an array, otherwise treat it as an object. Child classes may + // need to extend this method if this doesn't represent the data correctly. + + $treatAsObject = array_reduce( + array_keys($value), + function ($carry, $item) { + return (is_string($item) && $carry); + }, + true + ); + + $a = array(); + foreach ( $value as $k => $item ) { + $a[$k] = $this->_toStdClass($item); + } + return ( $treatAsObject ? (object) $a : array_values($a) ); + } elseif ( is_object($value) ) { + if ( $value instanceof iEntity ) { + return $value->toStdClass(); + } else { + // If Error, don't know how to convert the object + $this->logger->debug( + sprintf("Object '%s' does not implement iEntity, using get_object_vars() to convert to stdClass", get_class($value)) + ); + return (object) get_object_vars($value); + } + } else { + // This is a scalar value + return $value; + } + + } // _toStdClass() + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::compare() + * ------------------------------------------------------------------------------------------ + */ + + public function compare(iEntity $cmp) + { + return ( $this === $cmp ); + } // compare() + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::toJson() + * ------------------------------------------------------------------------------------------ + */ + + public function toJson() + { + return json_encode($this->toStdClass()); + } // toJson() + + /* ------------------------------------------------------------------------------------------ + * Reset all of the properties of this object to their default (unconfigured) + * values. This bypasses the usual checks enforced in filterAndVerifyValue() and + * __set(). + * + * @return This object + * ------------------------------------------------------------------------------------------ + */ + + public function resetPropertyValues() + { + foreach ( $this->defaultPropertyValues as $k => $v ) { + $this->properties[$k] = $v; + } + + return $this; + + } // resetPropertyValues() + + /* ------------------------------------------------------------------------------------------ + * Generic setter method for scalar properties. This method will set simple properties + * and perform input verification on individual properties, which works well for + * simple objects. If a more complex operation is required to initialize a composite, + * such as setting an array of objects that need to be instantiated then this method + * should be overriden in the child class. + * + * NOTE: This method SETS the value of a property, it does not add to it. Use an + * addXxx() method for that. + * + * NOTE: Setting a property to NULL essentially clears the value. If the cleared value + * should be something other than NULL (an array, for instance) it should be handled + * in a child class. + * + * @param $property The name of the property to set + * @param $value The new value of the property + * + * @throw Exception If a required parameter is given a null or empty value + * @throw Exception If a parameter fails verification + * ------------------------------------------------------------------------------------------ + */ + + public function __set($property, $value) + { + if ( array_key_exists($property, $this->properties) ) { + $this->properties[$property] = $this->filterAndVerifyValue($property, $value); + } else { + $this->logger->warning( + sprintf("%s: Attempt to set unsupported property: '%s'", get_class($this), $property) + ); + } + } // __set() + + /* ------------------------------------------------------------------------------------------ + * Generic getter method for properties not otherwise covered. + * + * @param $property The name of the property to retrieve + * + * @return The property, or NULL if the property doesn't exist. + * ------------------------------------------------------------------------------------------ + */ + + public function __get($property) + { + if ( array_key_exists($property, $this->properties) ) { + return $this->properties[$property]; + } else { + $this->logger->warning( + sprintf("%s: Attempt to access unsupported property: '%s'", get_class($this), $property) + ); + } + + return null; + } // __get() + + /* ------------------------------------------------------------------------------------------ + * Return TRUE if the property exists and is not NULL. + * + * @param $property The name of the property to retrieve + * + * @return TRUE if the property exists and is not NULL, FALSE otherwise. + * ------------------------------------------------------------------------------------------ + */ + + public function __isset($property) + { + return ( array_key_exists($property, $this->properties) && null !== $this->properties[$property] ); + } // __isset() + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::__toString() + * ------------------------------------------------------------------------------------------ + */ + + public function __toString() + { + return get_class($this); + } // __toString() +} // class Entity diff --git a/classes/ETL/DbModel/Index.php b/classes/ETL/DbModel/Index.php new file mode 100644 index 0000000000..1e5a14dc62 --- /dev/null +++ b/classes/ETL/DbModel/Index.php @@ -0,0 +1,207 @@ + + * @date 2015-10-29 + * + * @see Table + * @see iEntity + * ========================================================================================== + */ + +namespace ETL\DbModel; + +use Log; +use stdClass; + +class Index extends NamedEntity implements iEntity +{ + // Properties required by this class. These will be merged with other required + // properties up the call chain. See @Entity::$requiredProperties + private $localRequiredProperties = array( + 'columns' + ); + + // Properties provided by this class. These will be merged with other properties up + // the call chain. See @Entity::$properties + private $localProperties = array( + 'columns' => array(), + 'type' => null, + 'is_unique' => null + ); + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::__construct() + * ------------------------------------------------------------------------------------------ + */ + + public function __construct($config, $systemQuoteChar = null, Log $logger = null) + { + // Property merging is performed first so the values can be used in the constructor + parent::mergeProperties($this->localRequiredProperties, $this->localProperties); + parent::__construct($config, $systemQuoteChar, $logger); + } + + /* ------------------------------------------------------------------------------------------ + * @see aNamedEntity::initialize() + * ------------------------------------------------------------------------------------------ + */ + + public function initialize(stdClass $config) + { + // Local verifications + + if ( ! isset($config->name) ) { + $config->name = $this->generateIndexName($config->columns); + } + + parent::initialize($config); + + } // initialize() + + /* ------------------------------------------------------------------------------------------ + * @see Entity::filterAndVerifyValue() + * ------------------------------------------------------------------------------------------ + */ + + protected function filterAndVerifyValue($property, $value) + { + $value = parent::filterAndVerifyValue($property, $value); + + if ( null === $value ) { + return $value; + } + + switch ( $property ) { + + case 'is_unique': + $origValue = $value; + $value = \xd_utilities\filter_var($value, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); + if ( null === $value ) { + $this->logAndThrowException( + sprintf("%s must be a boolean, '%s' given", $property, gettype($origValue)) + ); + } + break; + + case 'columns': + if ( ! is_array($value) ) { + $this->logAndThrowException( + sprintf("%s must be an array, '%s' given", $property, gettype($value)) + ); + } elseif ( 0 == count($value) ) { + $this->logAndThrowException( + sprintf("%s must be an non-empty array", $property) + ); + } + break; + + case 'type': + if ( ! is_string($value) ) { + $this->logAndThrowException( + sprintf("%s name must be a string, '%s' given", $property, gettype($value)) + ); + } + break; + + default: + break; + } // switch ( $property ) + + return $value; + + } // filterAndVerifyValue() + + /* ------------------------------------------------------------------------------------------ + * Auto-generate an index name based on the columns included in the index. If the length of the + * index name would be too large use a hash. + * + * @param $columns The array of index column names + * + * @return The generated index name + * ------------------------------------------------------------------------------------------ + */ + + private function generateIndexName(array $columns) + { + $str = implode("_", $columns); + $name = ( strlen($str) <= 32 ? $str : md5($str) ); + return "index_" . $name; + } // generateIndexName() + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::compare() + * ------------------------------------------------------------------------------------------ + */ + + public function compare(iEntity $cmp) + { + if ( ! $cmp instanceof Index ) { + return 1; + } + + // Indexes are considered equal if all non-null properties are the same but only the name and + // columns are required but if the type and uniqueness are provided use those in the comparison + // as well. + + if ( $this->name != $cmp->name || $this->columns != $cmp->columns ) { + return -1; + } + + // The following properties have a default set by the database. If the property is not specified + // a value will be provided when the database information schema is queried. + + if ( ( null !== $this->type && null !== $cmp->type ) && $this->type != $cmp->type ) { + return -11; + } + + // The following properties do not have defaults set by the database and should be considered if + // one of them is set. + + // By default a primary key in MySQL has the name PRIMARY and is unique + + if ( "PRIMARY" != $this->name && "PRIMARY" != $cmp->name + && ( null !== $this->is_unique && null !== $cmp->is_unique ) + && $this->is_unique != $cmp->is_unique ) { + return -111; + } + + return 0; + + } // compare() + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::getSql() + * ------------------------------------------------------------------------------------------ + */ + + public function getSql($includeSchema = false) + { + // Primary keys always have an index name of "PRIMARY" + // See https://dev.mysql.com/doc/refman/5.7/en/create-table.html + + // Indexes may be created or altered in different ways (CREATE TABLE vs. ALTER TABLE) so we only + // return the essentials of the definition and let the Table class figure out the appropriate + // way to put them together. + + $parts = array(); + if ( null !== $this->name && "PRIMARY" == $this->name ) { + $parts[] = "PRIMARY KEY"; + } else { + $parts[] = ( null !== $this->is_unique && $this->is_unique ? "UNIQUE ": "" ) + . "INDEX " + . $this->getName(true); + } + + if ( null !== $this->type ) { + $parts[] = "USING " . $this->type; + } + + $parts[] = "(" . implode(", ", array_map(array($this, 'quote'), $this->columns)) . ")"; + + return implode(" ", $parts); + + } // getSql() +} // class Index diff --git a/classes/ETL/DbModel/Join.php b/classes/ETL/DbModel/Join.php new file mode 100644 index 0000000000..14189630f1 --- /dev/null +++ b/classes/ETL/DbModel/Join.php @@ -0,0 +1,116 @@ + + * @date 2017-04-28 + * + * @see Query + * @see iEntity + * ========================================================================================== + */ + +namespace ETL\DbModel; + +use Log; + +class Join extends SchemaEntity implements iEntity +{ + // Properties required by this class. These will be merged with other required + // properties up the call chain. See @Entity::$requiredProperties + private $localRequiredProperties = array(); + + // Properties provided by this class. These will be merged with other properties up + // the call chain. See @Entity::$properties + private $localProperties = array( + // Join type (e.g., "LEFT OUTER") + 'type' => null, + // Alias for the joined table + 'alias' => null, + // Join ON clause (not needed for FROM) + 'on' => null + ); + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::__construct() + * ------------------------------------------------------------------------------------------ + */ + + public function __construct($config, $systemQuoteChar = null, Log $logger = null) + { + // Property merging is performed first so the values can be used in the constructor + parent::mergeProperties($this->localRequiredProperties, $this->localProperties); + parent::__construct($config, $systemQuoteChar, $logger); + } // __construct() + + /* ------------------------------------------------------------------------------------------ + * @see Entity::filterAndVerifyValueValue() + * ------------------------------------------------------------------------------------------ + */ + + protected function filterAndVerifyValue($property, $value) + { + $value = parent::filterAndVerifyValue($property, $value); + + if ( null === $value ) { + return $value; + } + + switch ( $property ) { + case 'type': + case 'alias': + case 'on': + if ( ! is_string($value) ) { + $this->logAndThrowException( + sprintf("%s name must be a string, '%s' given", $property, gettype($value)) + ); + } + break; + + default: + break; + } // switch ( $property ) + + return $value; + } // filterAndVerifyValue() + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::compare() + * ------------------------------------------------------------------------------------------ + */ + + public function compare(iEntity $cmp) + { + if ( ! $cmp instanceof Join ) { + return 1; + } + + return ( $this == $cmp ); + + } // compare() + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::getSql() + * ------------------------------------------------------------------------------------------ + */ + + public function getSql($includeSchema = false) + { + $parts = array(); + + // Allow subqueries to be included and not quoted + $quoteName = ( 0 !== strpos($this->name, '(') ); + + $parts[] = ( null !== $this->schema && $includeSchema ? $this->getFullName() : $this->getName($quoteName) ); + if ( null !== $this->alias ) { + $parts[] = "AS " . $this->alias; + } + if ( null !== $this->on ) { + $parts[] = "ON " . $this->on; + } + + return implode(" ", $parts); + + } // getSql() +} // class Join diff --git a/classes/ETL/DbModel/NamedEntity.php b/classes/ETL/DbModel/NamedEntity.php new file mode 100644 index 0000000000..894dd60e30 --- /dev/null +++ b/classes/ETL/DbModel/NamedEntity.php @@ -0,0 +1,98 @@ + + * @date 2017-04-27 + * ========================================================================================== + */ + +namespace ETL\DbModel; + +use Log; +use ETL\DataEndpoint; +use ETL\DataEndpoint\DataEndpointOptions; + +class NamedEntity extends Entity +{ + // Properties required by this class. These will be merged with other required + // properties up the call chain. See @Entity::$requiredProperties + private $localRequiredProperties = array( + 'name' + ); + + // Properties provided by this class. These will be merged with other properties up + // the call chain. See @Entity::$properties + private $localProperties = array( + 'name' => null + ); + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::__construct() + * ------------------------------------------------------------------------------------------ + */ + + public function __construct($config, $systemQuoteChar = null, Log $logger = null) + { + // Property merging is performed first so the values can be used in the constructor + parent::mergeProperties($this->localRequiredProperties, $this->localProperties); + parent::__construct($config, $systemQuoteChar, $logger); + } + + /* ------------------------------------------------------------------------------------------ + * @see Entity::filterAndVerifyValue() + * ------------------------------------------------------------------------------------------ + */ + + protected function filterAndVerifyValue($property, $value) + { + $value = parent::filterAndVerifyValue($property, $value); + + if ( null === $value ) { + return $value; + } + + switch ( $property ) { + + case 'name': + if ( ! is_string($value) ) { + $this->logAndThrowException( + sprintf("%s must be a string, '%s' given", $property, gettype($value)) + ); + } + break; + + default: + break; + } // switch ( $property ) + + return $value; + + } // filterAndVerifyValue() + + /* ------------------------------------------------------------------------------------------ + * This is a convienece metod to return the model name, optionally quoted, rather than + * using $this->quote($obj->name). + * + * @param $quote true to wrap the name in quotes to handle special characters + * + * @return The name of this table, optionally quoted with the schema + * ------------------------------------------------------------------------------------------ + */ + + public function getName($quote = false) + { + return ( $quote ? $this->quote($this->name) : $this->name ); + } // getName() + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::__toString() + * ------------------------------------------------------------------------------------------ + */ + + public function __toString() + { + return sprintf('%s (%s)', $this->name, get_class($this)); + } // __toString() +} // class NamedEntity diff --git a/classes/ETL/DbModel/Query.php b/classes/ETL/DbModel/Query.php new file mode 100644 index 0000000000..1f52ff1549 --- /dev/null +++ b/classes/ETL/DbModel/Query.php @@ -0,0 +1,525 @@ + + * @date 2015-11-20 + * ========================================================================================== + */ + +namespace ETL\DbModel; + +use ETL\Utilities; +use Log; + +class Query extends Entity implements iEntity +{ + + // The overseer restriction values for this query. These are the templates that have + // been processed by the overseer to include the values based on overseer options. + protected $overseerRestrictionValues = array(); + + // Properties required by this class. These will be merged with other required + // properties up the call chain. See @Entity::$requiredProperties + private $localRequiredProperties = array( + 'records', + 'joins' + ); + + // Properties provided by this class. These will be merged with other properties up + // the call chain. See @Entity::$properties + private $localProperties = array( + // Records describing the fields used to populate the aggregation table + 'records' => array(), + + // Join tables. A single table generates the FROM clause while the rest are added as JOINS + 'joins' => array(), + + // Optional array of WHERE clauses + 'where' => array(), + + // Optional array of GROUP BY clauses + 'groupby' => array(), + + // Optional array of ORDER BY fields + 'orderby' => array(), + + // Optional defined macros + 'macros' => array(), + + // Query hints (See http://dev.mysql.com/doc/refman/5.7/en/query-cache-in-select.html) + 'query_hint' => null, + + // The list of ETL overseer restrictions supported by this query, as parsed from + // the query definition. Queries are not required to support restrictions and if a + // value for a restriction has not been set the restriction will not be + // applied. The ${VALUE} macro will be replaced by the value provided by the + // Overseer. For example: + // + // "source_query": { + // "overseer_restrictions": { + // "start_date": "jf.start_date >= ${VALUE}", + // "end_date": "jf.end_date <= ${VALUE}", + // "include_only_resource_codes": "jf.resource_id IN ${VALUE}", + // "exclude_resource_codes": "jf.resource_id NOT IN ${VALUE}" + // } + 'overseer_restrictions' => array() + ); + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::__construct() + * ------------------------------------------------------------------------------------------ + */ + + public function __construct($config, $systemQuoteChar = null, Log $logger = null) + { + // Property merging is performed first so the values can be used in the constructor + parent::mergeProperties($this->localRequiredProperties, $this->localProperties); + parent::__construct($config, $systemQuoteChar, $logger); + } // __construct() + + /* ------------------------------------------------------------------------------------------ + * @see Entity::filterAndVerifyValue() + * ------------------------------------------------------------------------------------------ + */ + + protected function filterAndVerifyValue($property, $value) + { + $value = parent::filterAndVerifyValue($property, $value); + + if ( null === $value ) { + return $value; + } + + switch ( $property ) { + case 'records': + case 'overseer_restrictions': + if ( ! is_object($value) ) { + $this->logAndThrowException( + sprintf("%s name must be an object, '%s' given", $property, gettype($value)) + ); + } + break; + + case 'where': + case 'groupby': + case 'orderby': + case 'macros': + case 'joins': + // Note that we are only checking that the value is an array here and not + // the array elements. That must come later. + + if ( ! is_array($value) ) { + $this->logAndThrowException( + sprintf("%s name must be an array, '%s' given", $property, gettype($value)) + ); + } + break; + + case 'query_hint': + if ( ! is_string($value) ) { + $this->logAndThrowException( + sprintf("%s name must be a string, '%s' given", $property, gettype($value)) + ); + } + break; + + default: + break; + } // switch ( $property ) + + return $value; + + } // filterAndVerifyValue() + + /* ------------------------------------------------------------------------------------------ + * Verify the query. Check that any columns referenced in the query are present in the + * destination table. + * + * @param Table $destinationTable The table that the query will be checked against. + * + * @see iEntity::verify() + * ------------------------------------------------------------------------------------------ + */ + + public function verify() + { + if ( 1 != func_num_args() ) { + $this->logAndThrowException( + sprintf('%s expected 1 argument, got %d', __FUNCTION__, func_num_args()) + ); + } + + $destinationTable = func_get_arg(0); + + if ( ! $destinationTable instanceof Table ) { + $this->logAndThrowException( + sprintf( + '%s expected object of type Table, got %s', + __FUNCTION__, + ( is_object($destinationTable) ? get_class($destinationTable) : gettype($destinationTable) ) + ) + ); + } + + $columnNames = $destinationTable->getColumnNames(); + $missingColumnNames = array_diff(array_keys($this->records), $columnNames); + + if ( 0 != count($missingColumnNames) ) { + $this->logAndThrowException("Columns in records not found in table: " . implode(", ", $missingColumnNames)); + } + + return true; + + } // verify() + + /* ------------------------------------------------------------------------------------------ + * Add (or overwrite) a record to this query. Records map column names to values in the SELECT statement. + * + * @param $columnName The column that the formula will be associated with. + * @param $formula The formula associated with the column. + * + * @return This object to support method chaining. + * + * @throw Exception If the column name does not exist. + * @throw Exception If the formula is empty. + * @throw Exception If the column name has already been specified. + * ------------------------------------------------------------------------------------------ + */ + + public function addRecord($columnName, $formula) + { + // Note in PHP "" and "0" are both equal to 0 due to conversion comparing strings to integers. + + if ( empty($columnName) ) { + $this->logAndThrowException("Empty column name"); + } + + if ( null === $formula ) { + $this->logAndThrowException(sprintf("Empty formula for column '%s'", $columnName)); + } + + $this->properties['records'][$columnName] = $formula; + + return $this; + + } // addRecord() + + /* ------------------------------------------------------------------------------------------ + * Get a formula for the specified column. + * + * @param $columnName The column to retrieve. + * + * @return The formula for the specified column, or false if none exists. + * ------------------------------------------------------------------------------------------ + */ + + public function getRecord($columnName) + { + return ( array_key_exists($columnName, $this->properties['records']) ? $this->properties['records'][$columnName] : false ); + } // getRecord() + + /* ------------------------------------------------------------------------------------------ + * Remove a column record if it exists and return the formula. + * + * @param $columnName The column to remove. + * + * @return The formula for the specified column, or FALSE if none exists. + * ------------------------------------------------------------------------------------------ + */ + + public function removeRecord($columnName) + { + $record = $this->getRecord($columnName); + if ( false !== $record ) { + unset($this->properties['records'][$columnName]); + } + return $record; + } // removeRecord() + + + /* ------------------------------------------------------------------------------------------ + * Add a join clause for this query. + * + * @param $definition An object containing the column definition, or an instantiated Join + * object to add + * + * @return This object to support method chaining. + * ------------------------------------------------------------------------------------------ + */ + + public function addJoin($config) + { + $item = ( is_object($config) && $config instanceof Join + ? $config + : new Join($config, $this->systemQuoteChar, $this->logger) ); + + $this->properties['joins'][] = $item; + + return $this; + } // addJoin() + + /* ------------------------------------------------------------------------------------------ + * Add an overseer restriction template to this query based on the parsed query definition. + * Note that at this point we don't know if the restrictions are valid (i.e., supported by the + * EtlOverseer). + * + * @param $restrictions The name of the restriction + * @param $template A template for the restriction where ${VALUE} will be replaced by the value + * + * @throws Exception if the restriction or the template are invalid + * + * @return This object to support method chaining. + * ------------------------------------------------------------------------------------------ + */ + + public function addOverseerRestriction($restriction, $template) + { + if ( ! is_string($restriction) || "" == $restriction ) { + $this->logAndThrowException("Overseer restriction key must be a non-empty string"); + } elseif ( ! is_string($template) || "" == $template ) { + $this->logAndThrowException("Overseer restriction template must be a non-empty string"); + } + + $this->properties['overseer_restrictions'][$restriction] = $template; + return $this; + + } // addOverseerRestriction() + + /* ------------------------------------------------------------------------------------------ + * Get the list of configured overseer restrictions. + * + * @return An associative array where the keys are restriction names and the values are the + * templates for those restrictions. + * ------------------------------------------------------------------------------------------ + */ + + public function getOverseerRestrictions() + { + return $this->properties['overseer_restrictions']; + } // getOverseerRestrictions() + + /* ------------------------------------------------------------------------------------------ + * Add an overseer restriction value to this query. This is the template that has been processed + * by the overseer. These values are kept separate from the other where clauses. + * + * @param $restrictions The name of the restriction + * @param $value A processed overseer restriction template + * + * @throws Exception if the restriction or the value are invalid + * + * @return The value of the specified restriction, or FALSE if the name was not found. + * ------------------------------------------------------------------------------------------ + */ + + public function addOverseerRestrictionValue($restriction, $value) + { + if ( ! is_string($restriction) || "" == $restriction ) { + $this->logAndThrowException("Overseer restriction key must be a non-empty string"); + } elseif ( ! is_string($value) || "" == $value ) { + $this->logAndThrowException("Overseer restriction template must be a non-empty string"); + } + + $this->overseerRestrictionValues[$restriction] = $value; + return $this; + + } // addOverseerRestrictionValue() + + /* ------------------------------------------------------------------------------------------ + * Add an overseer restriction value to this query. This is the template that has been processed + * by the overseer. These values are kept separate from the other where clauses. + * + * @param $restrictions The name of the restriction + * @param $value A processed overseer restriction template + * + * @throws Exception if the restriction or the value are invalid + * + * @return This object to support method chaining. + * ------------------------------------------------------------------------------------------ + */ + + public function getOverseerRestrictionValues() + { + return $this->overseerRestrictionValues; + } // getOverseerRestrictionValues() + + /* ------------------------------------------------------------------------------------------ + * Generate a string containing the select statement described by the configuration. This string + * may contain macros that will need to be replaced before execution. + * + * @param $includeSchema true to include the schema in the item name, if appropriate. + * + * @return An array comtaining the SQL required for altering this item. + * ------------------------------------------------------------------------------------------ + */ + + public function getSql($includeSchema = true) + { + if ( 0 == count($this->joins) ) { + $this->logAndThrowException("At least one join is required"); + } + + // Use the records to generate the SELECT columns + + $columnList = array(); + $thisObj = $this; + foreach ( $this->records as $columnName => $formula ) { + + // Do not quote field names because we may have functions in the query. -smg + $columnList[] = "$formula AS $columnName"; + } + + // Use the first join as the main FROM table, followined by other joins. + + $myJoins = $this->joins; + + $joinList = array(); + $joinList[] = "FROM " . $myJoins[0]->getSql($includeSchema); + + for ($i = 1; $i < count($myJoins); $i++) { + if ( null === $myJoins[$i]->on ) { + $this->logger->debug( + sprintf("Join clause for table '%s' does not provide ON condition", $myJoins[$i]->name) + ); + } + + // When we move to explictly marking the FROM clause this functionality may be moved + // into the Join class + + $joinType = $myJoins[$i]->type; + + // Handle various join types. STRAIGHT_JOIN is a mysql enhancement. + + $joinStr = "JOIN"; + + if ( "STRAIGHT" == $joinType ) { + $joinStr = "STRAIGHT_JOIN"; + } elseif (null !== $joinType) { + $joinStr = $joinType . " JOIN"; + } + + $joinList[] = $joinStr . " " . $myJoins[$i]->getSql($includeSchema); + } // for ( $i = 1; $i < count($this->joins); $i++ ) + + // Construct the SELECT statement + + // Merge in where clauses along with any overseer restrictions provided + $whereConditions = array_merge($this->where, $this->overseerRestrictionValues); + + $sql = "SELECT" .( null !== $this->query_hint ? " " . $this->query_hint : "" ) . "\n" . + implode(",\n", $columnList) . "\n" . + implode("\n", $joinList) . "\n" . + ( count($whereConditions) > 0 ? "WHERE " . implode("\nAND ", $whereConditions) . "\n" : "" ) . + ( count($this->groupby) > 0 ? "GROUP BY " . implode(", ", $this->groupby) : "" ) . + ( count($this->orderby) > 0 ? "ORDER BY " . implode(", ", $this->orderby) : "" ); + + // If any macros have been defined, process those macros now. Since macros can contain variables + // themselves, we will process the variables later. + + if (count($this->macros) > 0) { + foreach ( $this->macros as $macro ) { + $sql = Utilities::processMacro($sql, $macro); + } + } + + return $sql; + + } // getSql() + + /* ------------------------------------------------------------------------------------------ + * iEntity::toStdClass() + * ------------------------------------------------------------------------------------------ + */ + + public function toStdClass() + { + $data = parent::toStdClass(); + + // Overwrite arrays that are expected to be objects. If overseer_restrictions is + // an empty array Entity::_toStdClass() won't know that it should be an object. + + if ( is_array($data->overseer_restrictions) ) { + $data->overseer_restrictions = (object) $data->overseer_restrictions; + } + + return $data; + + } // toStdClasS() + + /* ------------------------------------------------------------------------------------------ + * @see Entity::__set() + * ------------------------------------------------------------------------------------------ + */ + + public function __set($property, $value) + { + // If we are not setting a property that is a special case, just call the main setter + + $specialCaseProperties = array('joins', 'records', 'overseer_restrictions'); + + if ( ! in_array($property, $specialCaseProperties) ) { + parent::__set($property, $value); + return; + } + + // Verify values prior to doing anything with them so we can make assumptions later. + + $value = $this->filterAndVerifyValue($property, $value); + + // Handle special cases. + + switch ($property) { + case 'joins': + // Clear the array no matter what, that way NULL is handled properly. + $this->properties[$property] = array(); + if ( null !== $value ) { + foreach ( $value as $item ) { + $this->properties[$property][] = + ( is_object($item) && $item instanceof Join + ? $item + : new Join($item, $this->systemQuoteChar, $this->logger) ); + } + } + break; + + case 'records': + // Clear the array no matter what, that way NULL is handled properly. + $this->properties[$property] = array(); + if ( null !== $value ) { + foreach ( $value as $column => $formula ) { + // Provide a method for adding and verifying more complex information + $this->addRecord($column, $formula); + } + } + break; + + case 'overseer_restrictions': + // Clear the array no matter what, that way NULL is handled properly. + $this->properties[$property] = array(); + $this->overseerRestrictionValues = array(); + if ( null !== $value ) { + foreach ( $value as $restriction => $template ) { + $this->addOverseerRestriction($restriction, $template); + } + } + break; + + default: + break; + } // switch($property) + + } // __set() +} // class Query diff --git a/classes/ETL/DbModel/SchemaEntity.php b/classes/ETL/DbModel/SchemaEntity.php new file mode 100644 index 0000000000..e21bfae6ed --- /dev/null +++ b/classes/ETL/DbModel/SchemaEntity.php @@ -0,0 +1,100 @@ + + * @date 2017-04-27 + * ========================================================================================== + */ + +namespace ETL\DbModel; + +use Log; +use ETL\Loggable; +use ETL\DataEndpoint; +use ETL\DataEndpoint\DataEndpointOptions; + +class SchemaEntity extends NamedEntity +{ + // Properties required by this class. These will be merged with other required + // properties up the call chain. See @Entity::$requiredProperties + private $localRequiredProperties = array(); + + // Properties provided by this class. These will be merged with other properties up + // the call chain. See @Entity::$properties + private $localProperties = array( + 'schema' => null + ); + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::__construct() + * ------------------------------------------------------------------------------------------ + */ + + public function __construct($config, $systemQuoteChar = null, Log $logger = null) + { + // Property merging is performed first so the values can be used in the constructor + parent::mergeProperties($this->localRequiredProperties, $this->localProperties); + parent::__construct($config, $systemQuoteChar, $logger); + } + + /* ------------------------------------------------------------------------------------------ + * @see Entity::filterAndVerifyValue() + * ------------------------------------------------------------------------------------------ + */ + + protected function filterAndVerifyValue($property, $value) + { + $value = parent::filterAndVerifyValue($property, $value); + + if ( null === $value ) { + return $value; + } + + switch ( $property ) { + + case 'schema': + if ( ! is_string($value) ) { + $this->logAndThrowException( + sprintf("%s must be a string, '%s' given", $property, gettype($value)) + ); + } + break; + + default: + break; + } // switch ( $property ) + + return $value; + + } // filterAndVerifyValue() + + /* ------------------------------------------------------------------------------------------ + * @param $quote TRUE if the schema and name should be quoted, defaults to TRUE. + * + * @return The fully qualified and quoted name including the schema, if one was set. + * ------------------------------------------------------------------------------------------ + */ + + public function getFullName($quote = true) + { + return ( null !== $this->schema ? $this->getSchema($quote) . "." : "" ) + . $this->getName($quote); + } // getFullName() + + /* ------------------------------------------------------------------------------------------ + * This is a convienece metod to return the model schema, optionally quoted, rather + * than using $this->quote($obj->schema). + * + * @param $quote true to wrap the name in quotes to handle special characters + * + * @return The name of this table, optionally quoted with the schema + * ------------------------------------------------------------------------------------------ + */ + + public function getSchema($quote = false) + { + return ( $quote ? $this->quote($this->schema) : $this->schema ); + } // getSchema() +} // class SchemaEntity diff --git a/classes/ETL/DbModel/Table.php b/classes/ETL/DbModel/Table.php new file mode 100644 index 0000000000..053efa1b8b --- /dev/null +++ b/classes/ETL/DbModel/Table.php @@ -0,0 +1,833 @@ + + * @date 2017-04-28 + * ========================================================================================== + */ + +namespace ETL\DbModel; + +use ETL\DataEndpoint\iRdbmsEndpoint; +use Log; +use stdClass; + +class Table extends SchemaEntity implements iEntity, iDiscoverableEntity, iAlterableEntity +{ + // Properties required by this class. These will be merged with other required + // properties up the call chain. See @Entity::$requiredProperties + private $localRequiredProperties = array( + 'columns' + ); + + // Properties provided by this class. These will be merged with other properties up + // the call chain. See @Entity::$properties + private $localProperties = array( + // Optional table comment + 'comment' => null, + + // Optional table engine + 'engine' => null, + + // Associative array where the keys are column names and the values are Column objects + 'columns' => array(), + + // Associative array where the keys are index names and the values are Index objects + 'indexes' => array(), + + // Associative array where the keys are trigger names and the values are Trigger objects + 'triggers' => array(), + ); + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::__construct() + * ------------------------------------------------------------------------------------------ + */ + + public function __construct($config, $systemQuoteChar = null, Log $logger = null) + { + // Property merging is performed first so the values can be used in the constructor + parent::mergeProperties($this->localRequiredProperties, $this->localProperties); + parent::__construct($config, $systemQuoteChar, $logger); + } // __construct() + + /* ------------------------------------------------------------------------------------------ + * @see aNamedEntity::initialize() + * ------------------------------------------------------------------------------------------ + */ + + public function initialize(stdClass $config) + { + // The table object only cares about the table definition but there may be other configuration keys present. Only continue on with the table definition. + + if ( isset($config->table_definition) ) { + parent::initialize($config->table_definition); + } else { + parent::initialize($config); + } + + } // initialize() + + /* ------------------------------------------------------------------------------------------ + * @see Entity::filterAndVerifyValue() + * ------------------------------------------------------------------------------------------ + */ + + protected function filterAndVerifyValue($property, $value) + { + $value = parent::filterAndVerifyValue($property, $value); + + if ( null === $value ) { + return $value; + } + + switch ( $property ) { + case 'columns': + case 'indexes': + case 'triggers': + // Note that we are only checking that the value is an array here and not + // the array elements. That must come later. + + if ( ! is_array($value) ) { + $this->logAndThrowException( + sprintf("%s must be an array, '%s' given", $property, gettype($value)) + ); + } + break; + + case 'comment': + case 'engine': + if ( ! is_string($value) ) { + $this->logAndThrowException( + sprintf("%s name must be a string, '%s' given", $property, gettype($value)) + ); + } + break; + + default: + break; + } // switch ( $property ) + + return $value; + + } // filterAndVerifyValue() + + /* ------------------------------------------------------------------------------------------ + * Verify the table by checking that any columns referenced in the indexes are present + * in the column definitions. + * + * @see iEntity::verify() + * + * @throws Exception If there are errors during validation + * ------------------------------------------------------------------------------------------ + */ + + public function verify() + { + // Verify index columns match table columns + + $columnNames = $this->getColumnNames(); + + foreach ( $this->indexes as $index ) { + $missingColumnNames = array_diff($index->columns, $columnNames); + if ( 0 != count($missingColumnNames) ) { + $this->logAndThrowException( + sprintf("Columns in index '%s' not found in table definition: %s", $index->name, implode(", ", $missingColumnNames)) + ); + } + } // foreach ( $this->indexes as $index ) + + return true; + + } // verify() + + /* ------------------------------------------------------------------------------------------ + * Discover a Table using the database information schema and populate this object + * with the result. When creating a Table to be discovered, pass a NULL $config + * object to the constructor. + * + * @param string $source The name of the table to discover + * @param iRdbmsEndpoint $endpoint The DataEndpoint used to connect to the database + * (provides schema) + * + * @see iDiscoverable::discover() + * ------------------------------------------------------------------------------------------ + */ + + public function discover($source) + { + if ( 2 != func_num_args() ) { + $this->logAndThrowException( + sprintf('%s expected 2 arguments, got %d', __FUNCTION__, func_num_args()) + ); + } + + $endpoint = func_get_arg(1); + + if ( ! $endpoint instanceof iRdbmsEndpoint ) { + $this->logAndThrowException( + sprintf( + '%s expected object implementing iRdbmsEndpoint, got %s', + __FUNCTION__, + ( is_object($endpoint) ? get_class($endpoint) : gettype($endpoint) ) + ) + ); + } + + $this->resetPropertyValues(); + + $schemaName = null; + $qualifiedTableName = null; + $systemQuoteChar = $endpoint->getSystemQuoteChar(); + + // If a schema was specified in the table name use it, otherwise use the default schema + + if ( false === strpos($source, ".") ) { + $schemaName = $endpoint->getSchema(); + $qualifiedTableName = sprintf('%s.%s', $schemaName, $source); + } else { + $qualifiedTableName = $source; + list($schemaName, $source) = explode(".", $source); + } + + $params = array(':schema' => $schemaName, + ':tablename' => $source); + + $this->logger->debug("Discover table '$qualifiedTableName'"); + + // Query table properties + + $sql = "SELECT +engine, table_comment as comment +FROM information_schema.tables +WHERE table_schema = :schema +AND table_name = :tablename"; + + try { + $result = $endpoint->getHandle()->query($sql, $params); + if ( count($result) > 1 ) { + $this->logAndThrowException("Multiple rows returned for table '$qualifiedTableName'"); + } + + // The table did not exist, return false + + if ( 0 == count($result) ) { + return false; + } + + } catch (Exception $e) { + $this->logAndThrowException("Error discovering table '$qualifiedTableName': " . $e->getMessage()); + } + + $row = array_shift($result); + + $this->name = $source; + $this->schema = $schemaName; + $this->engine = $row['engine']; + $this->comment = $row['comment']; + + // Query columns. Querying for the default needs some explaining. The information schema stores + // the default as null unless one was specifically provided so we need some logic to get things + // into the shape we want. + + // SMG: We should do a better job of detecting equivalent columns. For example "int unsigned" is + // equivalent to "int(10) unsigned". + + $sql = "SELECT +column_name as name, column_type as type, is_nullable as nullable, +column_default as " . $endpoint->quoteSystemIdentifier("default") . ", +IF('' = extra, NULL, extra) as extra, +IF('' = column_comment, NULL, column_comment) as " . $endpoint->quoteSystemIdentifier("comment") . " +FROM information_schema.columns +WHERE table_schema = :schema +AND table_name = :tablename +ORDER BY ordinal_position ASC"; + + try { + $result = $endpoint->getHandle()->query($sql, $params); + if ( 0 == count($result) ) { + $this->logAndThrowException("No columns returned for table '$qualifiedTableName'"); + } + } catch (Exception $e) { + $this->logAndThrowException("Error discovering table '$qualifiedTableName' columns: " . $e->getMessage()); + } + + foreach ( $result as $row ) { + $this->addColumn((object) $row); + } + + // Query indexes. + + $sql = "SELECT +index_name as name, index_type as " . $endpoint->quoteSystemIdentifier("type") . ", (non_unique = 0) as is_unique, +GROUP_CONCAT(column_name ORDER BY seq_in_index ASC) as columns +FROM information_schema.statistics +WHERE table_schema = :schema +AND table_name = :tablename +GROUP BY index_name +ORDER BY index_name ASC"; + + try { + $result = $endpoint->getHandle()->query($sql, $params); + } catch (Exception $e) { + $this->logAndThrowException("Error discovering table '$qualifiedTableName' indexes: " . $e->getMessage()); + } + + foreach ( $result as $row ) { + $row['columns'] = explode(",", $row['columns']); + $this->addIndex((object) $row); + } + + // Query triggers + + $sql = "SELECT +trigger_name as name, action_timing as time, event_manipulation as event, +event_object_schema as " . $endpoint->quoteSystemIdentifier("schema") . ", event_object_table as " . $endpoint->quoteSystemIdentifier("table") . ", definer, +action_statement as body +FROM information_schema.triggers +WHERE event_object_schema = :schema +and event_object_table = :tablename +ORDER BY trigger_name ASC"; + + try { + $result = $endpoint->getHandle()->query($sql, $params); + } catch (Exception $e) { + $this->logAndThrowException("Error discovering table '$qualifiedTableName' triggers: " . $e->getMessage()); + } + + foreach ( $result as $row ) { + $this->addTrigger((object) $row); + } + + return true; + + } // discover() + + /* ------------------------------------------------------------------------------------------ + * Add a column to this table. + * + * @param $config An object containing the column definition, or a Column object to add + * @param $overwriteDuplicates TRUE to allow overwriting of duplicate column names. If false, throw + * an exception if a duplicate column is added. + * + * @return This object to support method chaining. + * + * @throw Exception if the new item has the same name as an existing item and + * overwrite is not TRUE + * ------------------------------------------------------------------------------------------ + */ + + public function addColumn($config, $overwriteDuplicates = false) + { + $item = ( is_object($config) && $config instanceof Column + ? $config + : new Column($config, $this->systemQuoteChar, $this->logger) ); + + if ( array_key_exists($item->name, $this->columns) && ! $overwriteDuplicates ) { + $this->logAndThrowException( + sprintf("Cannot add duplicate column '%s'", $item->name), + array('log_level' => PEAR_LOG_WARNING) + ); + } + + $this->properties['columns'][$item->name] = $item; + + return $this; + + } // addColumn() + + /* ------------------------------------------------------------------------------------------ + * Get the list of column names. + * + * @return An array of column names. + * ------------------------------------------------------------------------------------------ + */ + + public function getColumnNames() + { + return array_keys($this->columns); + } // getColumnNames() + + /* ------------------------------------------------------------------------------------------ + * Get a Column object with the specified name. + * + * @param $name The column to retrieve. + * + * @return The Column object with the specified name or FALSE if the trigger does not exist + * ------------------------------------------------------------------------------------------ + */ + + public function getColumn($name) + { + return ( array_key_exists($name, $this->columns) ? $this->properties['columns'][$name] : false ); + } // getColumn() + + /* ------------------------------------------------------------------------------------------ + * Add an index to this table. + * + * @param $config An object containing the column definition, or a Column object to add + * @param $overwriteDuplicates TRUE to allow overwriting of duplicate column names. If false, throw + * an exception if a duplicate column is added. + * + * @return This object to support method chaining. + * + * @throw Exception if the new item has the same name as an existing item and + * overwrite is not TRUE + * ------------------------------------------------------------------------------------------ + */ + + public function addIndex($config, $overwriteDuplicates = false) + { + $item = ( is_object($config) && $config instanceof Index + ? $config + : new Index($config, $this->systemQuoteChar, $this->logger) ); + + if ( array_key_exists($item->name, $this->indexes) && ! $overwriteDuplicates ) { + $this->logAndThrowException( + sprintf("Cannot add duplicate index '%s'", $item->name) + ); + } + + $this->properties['indexes'][$item->name] = $item; + + return $this; + + } // addIndex() + + /* ------------------------------------------------------------------------------------------ + * Get the list of column names. + * + * @return An array of column names. + * ------------------------------------------------------------------------------------------ + */ + + public function getIndexNames() + { + return array_keys($this->indexes); + } // getIndexNames() + + /* ------------------------------------------------------------------------------------------ + * Get an Index object with the specified name. + * + * @param $name The name of the index to retrieve. + * + * @return The Index object with the specified name or FALSE if the trigger does not exist + * ------------------------------------------------------------------------------------------ + */ + + public function getIndex($name) + { + return ( array_key_exists($name, $this->indexes) ? $this->properties['indexes'][$name] : false ); + } // getIndex() + + /* ------------------------------------------------------------------------------------------ + * Add a trigger to this table. + * + * @param $config An object containing the column definition, or a Column object to add + * @param $overwriteDuplicates TRUE to allow overwriting of duplicate column names. If false, throw + * an exception if a duplicate column is added. + * + * @return This object to support method chaining. + * + * @throw Exception if the new item has the same name as an existing item and + * overwrite is not TRUE + * ------------------------------------------------------------------------------------------ + */ + + public function addTrigger($config, $overwriteDuplicates = false) + { + + $item = ( is_object($config) && $config instanceof Trigger + ? $config + : new Trigger($config, $this->systemQuoteChar, $this->logger) ); + + if ( array_key_exists($item->name, $this->triggers) && ! $overwriteDuplicates ) { + $this->logAndThrowException( + sprintf("Cannot add duplicate trigger '%s'", $item->name) + ); + } + + $this->properties['triggers'][$item->name] = $item; + + return $this; + + } // addTrigger() + + /* ------------------------------------------------------------------------------------------ + * Get the list of trigger names. + * + * @return An array of trigger names. + * ------------------------------------------------------------------------------------------ + */ + + public function getTriggerNames() + { + return array_keys($this->triggers); + } // getTriggerNames() + + /* ------------------------------------------------------------------------------------------ + * Get a Trigger object with the specified name. + * + * @param $name The trigger to retrieve. + * + * @return The Trigger object with the specified name or FALSE if the trigger does not exist + * ------------------------------------------------------------------------------------------ + */ + + public function getTrigger($name) + { + return ( array_key_exists($name, $this->triggers) ? $this->properties['triggers'][$name] : false ); + } // getTrigger() + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::getSql() + * ------------------------------------------------------------------------------------------ + */ + + public function getSql($includeSchema = true) + { + if ( null === $this->name || 0 == count($this->columns) ) { + return false; + } + + // Note: By using the name as the key, duplicate item names will use the last definition. This + // can occur when creating aggregation tables that contain a "year" column and an + // ${AGGREGATION_UNIT} column with an aggregation unit of "year". + + $columnCreateList = array(); + foreach ( $this->columns as $name => $column ) { + $columnCreateList[$name] = $column->getSql($includeSchema); + } + + $indexCreateList = array(); + foreach ( $this->indexes as $name => $index ) { + $indexCreateList[$name] = $index->getSql($includeSchema); + } + + $triggerCreateList = array(); + foreach ( $this->triggers as $name => $trigger ) { + // The table schema may have been set after the table was initially created. If the trigger + // doesn't explicitly define a schema, default to the table's schema. + if ( null === $trigger->schema ) { + $trigger->schema = $this->schema; + } + $triggerCreateList[$name] = $trigger->getSql($includeSchema); + } + + $tableName = ( $includeSchema ? $this->getFullName() : $this->getName(true) ); + + $sqlList = array(); + $sqlList[] = "CREATE TABLE IF NOT EXISTS $tableName (\n" . + " " . implode(",\n ", $columnCreateList) . + ( 0 != count($indexCreateList) ? ",\n " . implode(",\n ", $indexCreateList) : "" ) . "\n" . + ")" . + ( null !== $this->engine ? " ENGINE = " . $this->engine : "" ) . + ( null !== $this->comment && ! empty($this->comment) ? " COMMENT = '" . addslashes($this->comment) . "'" : "" ) . + ";"; + + foreach ( $triggerCreateList as $trigger ) { + $sqlList[] = $trigger; + } + + return $sqlList; + + } // getSql() + + /* ------------------------------------------------------------------------------------------ + * @param Table $destination The desired Table definition + * + * @see iAlterableEntity::getAlterSql() + * ------------------------------------------------------------------------------------------ + */ + + public function getAlterSql($destination, $includeSchema = true) + { + if ( ! $destination instanceof Table ) { + $this->logAndThrowException( + sprintf( + '%s expected Table object, got %s', + __FUNCTION__, + ( is_object($destination) ? get_class($destination) : gettype($destination) ) + ) + ); + } + + $alterList = array(); + $triggerList = array(); + + // Update names/docs to be clearer. We are migrating $this to $dest. + + // -------------------------------------------------------------------------------- + // Process columns + + $currentColNames = $this->getColumnNames(); + $destColNames = $destination->getColumnNames(); + + // Columns to be dropped, added, changed, or renamed + $dropColNames = array_diff($currentColNames, $destColNames); + $addColNames = array_diff($destColNames, $currentColNames); + $changeColNames = array_intersect($currentColNames, $destColNames); + $renameColNames = array(); + + // When renaming a column, be smart about it or a simple rename will mean that the new column is + // added and the old column is dropped causing potential data loss. Check for any processing + // hints on new columns: If a column is to be added, and there is a "rename_from" hint that + // matches an existing column name, mark this column to be renamed instead of added and dropped. + // We can then construct the CHANGE COLUMN statement. + + foreach ( $addColNames as $index => $addName ) { + $hint = $destination->getColumn($addName)->hints; + if ( null !== $hint + && isset($hint->rename_from) + && false !== ( $hintIndex = array_search($hint->rename_from, $dropColNames) ) ) + { + $renameColNames[$hint->rename_from] = $addName; + unset($addColNames[$index]); + unset($dropColNames[$hintIndex]); + } + } // foreach ( $addColNames as $addName ) + + if ( $this->engine != $destination->engine ) { + $alterList[] = "ENGINE = " . $destination->engine; + } + + if ( $this->comment != $destination->comment ) { + $alterList[] = "COMMENT = '" . addslashes($destination->comment) . "'"; + } + + foreach ( $addColNames as $name ) { + $alterList[] = "ADD COLUMN " . $destination->getColumn($name)->getSql($includeSchema); + } + + foreach ( $dropColNames as $name ) { + $alterList[] = "DROP COLUMN " . $this->quote($name); + } + + foreach ( $changeColNames as $name ) { + $destColumn = $destination->getColumn($name); + // Not all properties are required so a simple object comparison isn't possible + if ( 0 == $destColumn->compare($this->getColumn($name)) ) { + continue; + } + $alterList[] = "CHANGE COLUMN " . $destColumn->getName(true) . " " . $destColumn->getSql($includeSchema); + } + + foreach ( $renameColNames as $fromColumnName => $toColumnName ) { + $destColumn = $destination->getColumn($toColumnName); + $currentColumn = $this->getColumn($fromColumnName); + // Not all properties are required so a simple object comparison isn't possible + if ( 0 == $destColumn->compare($currentColumn) ) { + continue; + } + $alterList[] = "CHANGE COLUMN " . $currentColumn->getName(true) . " " . $destColumn->getSql($includeSchema); + } + + // -------------------------------------------------------------------------------- + // Processes indexes + + $currentIndexNames = $this->getIndexNames(); + $destIndexNames = $destination->getIndexNames(); + + $dropIndexNames = array_diff($currentIndexNames, $destIndexNames); + $addIndexNames = array_diff($destIndexNames, $currentIndexNames); + $changeIndexNames = array_intersect($currentIndexNames, $destIndexNames); + + foreach ( $dropIndexNames as $name ) { + $alterList[] = "DROP INDEX " . $this->quote($name); + } + + foreach ( $addIndexNames as $name ) { + $alterList[] = "ADD " . $destination->getIndex($name)->getSql($includeSchema); + } + + // Altered indexes need to be dropped then added + foreach ( $changeIndexNames as $name ) { + $destIndex = $destination->getIndex($name); + // Not all properties are required so a simple object comparison isn't possible + if ( 0 == $destIndex->compare($this->getIndex($name)) ) { + continue; + } + $alterList[] = "DROP INDEX " . $destIndex->getName(true); + $alterList[] = "ADD " . $destIndex->getSql($includeSchema); + } + + // -------------------------------------------------------------------------------- + // Process triggers + + // The table schema may have been set after the table was initially created. If the trigger + // doesn't explicitly define a schema, default to the table's schema. + + // if ( null === $trigger->getSchema() ) $trigger->setSchema($this->getSchema()); + + $currentTriggerNames = $this->getTriggerNames(); + $destTriggerNames = $destination->getTriggerNames(); + + $dropTriggerNames = array_diff($currentTriggerNames, $destTriggerNames); + $addTriggerNames = array_diff($destTriggerNames, $currentTriggerNames); + $changeTriggerNames = array_intersect($currentTriggerNames, $destTriggerNames); + + // Drop triggers first, then alter, then create + + foreach ( $dropTriggerNames as $name ) { + $triggerList[] = "DROP TRIGGER " . + ( null !== $this->schema && $includeSchema ? $this->quote($this->schema) . "." : "" ) . + $this->quote($name) . ";"; + } + + foreach ( $changeTriggerNames as $name ) { + $destTrigger = $destination->getTrigger($name); + if ( 0 == $destTrigger->compare($this->getTrigger($name))) { + continue; + } + + $triggerList[] = "DROP TRIGGER " . + ( null !== $this->schema && $includeSchema ? $this->quote($this->schema) . "." : "" ) . + $this->quote($name) . ";"; + $triggerList[] = $destination->getTrigger($name)->getSql($includeSchema); + } + + foreach ( $addTriggerNames as $name ) { + $triggerList[] = $destination->getTrigger($name)->getSql($includeSchema); + } + + // -------------------------------------------------------------------------------- + // Put it all together + + if ( 0 == count($alterList) && 0 == count($triggerList) ) { + return false; + } + + $tableName = ( $includeSchema ? $this->getFullName() : $this->getName(true) ); + + $sqlList = array(); + if ( 0 != count($alterList) ) { + $sqlList[] = "ALTER TABLE $tableName\n" . + implode(",\n", $alterList) . ";"; + } + + if ( 0 != count($triggerList) ) { + foreach ( $triggerList as $trigger ) { + $sqlList[] = $trigger; + } + } + + return $sqlList; + + } // getAlterSql() + + + /* ------------------------------------------------------------------------------------------ + * iEntity::toStdClass() + * ------------------------------------------------------------------------------------------ + */ + + public function toStdClass() + { + $data = parent::toStdClass(); + + // When we add columns, indexes, and triggers to a table we add them as an + // associative array where the keys are the column names. When generating the + // config With string keys Entity::_toStdClass() will assume an object because the + // keys are strings so convert them to arrays here. + + $data->columns = array_values((array) $data->columns); + $data->indexes = array_values((array) $data->indexes); + $data->triggers = array_values((array) $data->triggers); + + return $data; + + } // toStdClass() + + /* ------------------------------------------------------------------------------------------ + * @see Entity::__set() + * ------------------------------------------------------------------------------------------ + */ + + public function __set($property, $value) + { + // If we are not setting a property that is a special case, just call the main setter + $specialCaseProperties = array('columns', 'indexes', 'triggers'); + + if ( ! in_array($property, $specialCaseProperties) ) { + parent::__set($property, $value); + return; + } + + // Verify values prior to doing anything with them + + $value = $this->filterAndVerifyValue($property, $value); + + // Handle special cases. + + switch ($property) { + case 'columns': + $this->properties[$property] = array(); + // Clear the array no matter what, that way NULL is handled properly. + if ( null !== $value ) { + foreach ( $value as $item ) { + $column = ( is_object($item) && $item instanceof Column + ? $item + : new Column($item, $this->systemQuoteChar, $this->logger) ); + $this->properties[$property][$column->name] = $column; + } + } + break; + + case 'indexes': + $this->properties[$property] = array(); + // Clear the array no matter what, that way NULL is handled properly. + if ( null !== $value ) { + foreach ( $value as $item ) { + $index = ( is_object($item) && $item instanceof Index + ? $item + : new Index($item, $this->systemQuoteChar, $this->logger) ); + $this->properties[$property][$index->name] = $index; + } + } + break; + + case 'triggers': + $this->properties[$property] = array(); + // Clear the array no matter what, that way NULL is handled properly. + if ( null !== $value ) { + foreach ( $value as $item ) { + if ( is_object($item) && $item instanceof Trigger ) { + $this->properties[$property][$item->name] = $item; + } else { + if ( $item instanceof stdClass ) { + // Default to the current table name and schema of the parent table. + if ( ! isset($item->table) ) { + $item->table = $this->name; + } + if ( ! isset($item->schema) ) { + $item->schema = $this->schema; + } + } + $trigger = new Trigger($item, $this->systemQuoteChar, $this->logger); + $this->properties[$property][$trigger->name] = $trigger; + } + } + } + break; + + default: + break; + } // switch($property) + + } // __set() +} // class Table diff --git a/classes/ETL/DbModel/Trigger.php b/classes/ETL/DbModel/Trigger.php new file mode 100644 index 0000000000..2f1859d4d4 --- /dev/null +++ b/classes/ETL/DbModel/Trigger.php @@ -0,0 +1,162 @@ + + * @date 2017-04-28 + * + * @see Table + * @see iEntity + * ========================================================================================== + */ + +namespace ETL\DbModel; + +use Log; + +class Trigger extends SchemaEntity implements iEntity +{ + // Properties required by this class. These will be merged with other required + // properties up the call chain. See @Entity::$requiredProperties + private $localRequiredProperties = array( + 'time', + 'event', + 'table', + 'body' + ); + + // Properties provided by this class. These will be merged with other properties up + // the call chain. See @Entity::$properties + private $localProperties = array( + // The time that the trigger is fired (before, after) + 'time' => null, + // The event that the trigger is fired on (insert, update, delete) + 'event' => null, + // The table that the trigger is associated with + 'table' => null, + // The body of the trigger + 'body' => null, + // The trigger definer for ACL purposes + 'definer' => null + ); + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::__construct() + * ------------------------------------------------------------------------------------------ + */ + + public function __construct($config, $systemQuoteChar = null, Log $logger = null) + { + // Property merging is performed first so the values can be used in the constructor + parent::mergeProperties($this->localRequiredProperties, $this->localProperties); + parent::__construct($config, $systemQuoteChar, $logger); + } // __construct() + + /* ------------------------------------------------------------------------------------------ + * @see Entity::filterAndVerifyValue() + * ------------------------------------------------------------------------------------------ + */ + + protected function filterAndVerifyValue($property, $value) + { + $value = parent::filterAndVerifyValue($property, $value); + + if ( null === $value ) { + return $value; + } + + switch ( $property ) { + case 'time': + case 'event': + case 'table': + case 'body': + if ( ! is_string($value) ) { + $this->logAndThrowException( + sprintf("%s name must be a string, '%s' given", $property, gettype($value)) + ); + } + break; + + default: + break; + } // switch ( $property ) + + return $value; + + } // filterAndVerifyValue() + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::compare() + * ------------------------------------------------------------------------------------------ + */ + + public function compare(iEntity $cmp) + { + if ( ! $cmp instanceof Trigger ) { + return 1; + } + + // Schemas are optional for the trigger + + // Triggers are considered equal if all non-null properties are the same. + + if ( $this->name != $cmp->name + || $this->time != $cmp->time + || $this->event != $cmp->event + || $this->table != $cmp->table + || $this->body != $cmp->body ) { + return -1; + } + + // The following properties have a default set by the database. If the property is not specified + // a value will be provided when the database information schema is queried. + + if ( ( null !== $this->definer && null !== $cmp->definer ) && $this->definer != $cmp->definer ) { + return -1; + } + + // The following properties do not have defaults set by the database and should be considered if + // one of them is set. + + if ( ( null !== $this->schema || null !== $cmp->schema ) && $this->schema != $cmp->schema ) { + return -1; + } + + } // compare() + + /* ------------------------------------------------------------------------------------------ + * @see iEntity::getSql() + * ------------------------------------------------------------------------------------------ + */ + + public function getSql($includeSchema = false) + { + // Triggers queried from MySQL contain the begin/end but the body in the JSON may or may not. + + $addBeginEnd = ( 0 !== strpos($this->body, "BEGIN") ); + $name = ( $includeSchema ? $this->getFullName() : $this->getName(true) ); + $tableName = ( null !== $this->schema && $includeSchema ? $this->quote($this->schema) . "." : "" ) . + $this->quote($this->table); + $parts = array(); + $parts[] = "CREATE"; + if ( null !== $this->definer ) { + $parts[] = "DEFINER = " . $this->definer; + } + $parts[] = "TRIGGER $name"; + $parts[] = $this->time; + $parts[] = $this->event; + $parts[] = "ON $tableName FOR EACH ROW\n"; + if ( $addBeginEnd ) { + $parts[] = "BEGIN\n"; + } + $parts[] = $this->body; + if ( $addBeginEnd ) { + $parts[] = "\nEND"; + } + + return implode(" ", $parts); + + } // getSql() +} // class Trigger diff --git a/classes/ETL/DbModel/iAlterableEntity.php b/classes/ETL/DbModel/iAlterableEntity.php new file mode 100644 index 0000000000..ca0192e4f4 --- /dev/null +++ b/classes/ETL/DbModel/iAlterableEntity.php @@ -0,0 +1,35 @@ + + * @date 2017-05-04 + * ========================================================================================== + */ + +namespace ETL\DbModel; + +interface iAlterableEntity +{ + /* ------------------------------------------------------------------------------------------ + * Generate SQL to transform the this entity into the destination entity. For example, + * generate the ALTER TABLE statements required to transform this Table into the + * destination Table. More than one statement may be returned if required. For + * example, triggers are handled separately from the ALTER TABLE statement. The + * $destination and $includeSchema parameters are the minimum information needed for + * discovery and it is likely that additional parameters will be needed, but this is + * up to the implementation to specify. + * + * @param mixed $destination The entity that containing a defintion that we would like + * to achieve. + * @param bool $includeSchema TRUE to include the schema in the entity names, if + * appropriate. + * + * @return An array comtaining all SQL statements required for altering this item. + * + * @throw Exception If there is an error during the process + * ------------------------------------------------------------------------------------------ + */ + + public function getAlterSql($destination, $includeSchema = true); +} // interface iAlterableEntity diff --git a/classes/ETL/DbModel/iDiscoverableEntity.php b/classes/ETL/DbModel/iDiscoverableEntity.php new file mode 100644 index 0000000000..2c338ddf61 --- /dev/null +++ b/classes/ETL/DbModel/iDiscoverableEntity.php @@ -0,0 +1,35 @@ + + * @date 2017-05-04 + * ========================================================================================== + */ + +namespace ETL\DbModel; + +interface iDiscoverableEntity +{ + /* ------------------------------------------------------------------------------------------ + * Examine the source and build a prepresentation of the entity in this object based + * on the information found. If this object already contained a representation, it + * will be removed and replaced with the discovered representation. The $source + * parameter is the minimum information needed for discovery and it is likely that + * additional parameters will be needed, but this is up to the implementation to + * specify. + * + * @param mixed $source The source that we will query. This could be a file path or a + * table name. + * + * @return TRUE on success + * + * @throw Exception If there is an error during the discovery process + * ------------------------------------------------------------------------------------------ + */ + + public function discover($source); +} // interface iDiscoverableEntity diff --git a/classes/ETL/DbModel/iEntity.php b/classes/ETL/DbModel/iEntity.php new file mode 100644 index 0000000000..dc610d09ae --- /dev/null +++ b/classes/ETL/DbModel/iEntity.php @@ -0,0 +1,150 @@ + + * @date 2017-04-28 + * ========================================================================================== + */ + +namespace ETL\DbModel; + +use Log; +use stdClass; + +interface iEntity +{ + /* ------------------------------------------------------------------------------------------ + * The contructor MUST provide a configuration specification (or null if configuration + * will be handled manually). The type and verification of the specification is up to + * the implementation, but the default is a stdClass object. Additional arguments MAY + * be provided and will be handled by the individual contructors using func_get_args() + * + * @param mixed $config A representation of the configuration containing the item + * definition, or possibly a file name if supported by the particular item. If + * $config is NULL, an object with no property values should be created. + * @param string $systemQuoteChar Character used for escaping system identifiers. + * @param Log $logger PEAR Log object for system logging + * + * @throw Exception If an invalid nummber of arguments was provided + * @throw Exception If the column definition was incomplete + * ------------------------------------------------------------------------------------------ + */ + + public function __construct($config, $systemQuoteChar = null, Log $logger = null); + + /* ------------------------------------------------------------------------------------------ + * Initialize the object properties from a stdClass object. Only supported properties + * will be set. + * + * @param stdClass $config An object containing (property, value) pairs to be set. + * + * @return This object + * ------------------------------------------------------------------------------------------ + */ + + public function initialize(stdClass $config); + + /* ------------------------------------------------------------------------------------------ + * Reset all of the data properties of this object to their default (unconfigured) + * values. This bypasses the usual checks enforced in filterAndVerifyValue() and + * __set(). + * + * @return This object + * ------------------------------------------------------------------------------------------ + */ + + public function resetPropertyValues(); + + /* ------------------------------------------------------------------------------------------ + * Perform any necessary verification on the entity after it has been initialized. For + * simple entities, verificaiton may be a no-op. For example, table verification may + * entail checking that columns specified in index definitions are present in the + * table definition and Query verification may check that requested columns are + * present in the destination table. + * + * NOTE: The interface simple requires that this method is define and does not take + * any parameters. The implementation may require parameters and use + * function_get_args() to access them. + * + * @return TRUE if verification was successful, FALSE otherwise. + * ------------------------------------------------------------------------------------------ + */ + + public function verify(); + + /* ------------------------------------------------------------------------------------------ + * Compare the specified entity to this one and return 0 if the entities are the same, + * -1 if the specified entity is considered less than this one, or 1 of the specified + * entity is considered greater than this one. Not all entities support the concept of + * greater or less than in which case any non-zero value is considered different. + * + * @return 0 of the entities are the same, -1 if the specified entity is considered + * less than this one, or 1 of the specified entity is considered greater than this + * one. + * ------------------------------------------------------------------------------------------ + */ + + public function compare(iEntity $cmp); + + /* ------------------------------------------------------------------------------------------ + * Wrap a system identifier in quotes appropriate for the endpint if it is not already + * quoted. For example, MySQL uses a backtick (`) to quote identifiers while Oracle + * and Postgres using double quotes ("). + * + * @param $identifier A system identifier (schema, table, column name) to quote + * + * @return The identifier quoted appropriately for the endpoint + * ------------------------------------------------------------------------------------------ + */ + + public function quote($identifier); + + /* ------------------------------------------------------------------------------------------ + * Generate a simple representation of this entity's data properties as a stdClass + * object suitable for manipulating and feeding back into this entity's constructor as + * a configuration object. Essentially, re-create the configuration object and any + * changes. This can also be used to convert the object into a JSON + * representation. Any complex objects contained in the properties should implement + * iEntity so their toStdClass() method can be called to convert to stdClass, + * otherwise get_object_vars() may be used instead. + * + * @return A stdClass object representation for this entity + * ------------------------------------------------------------------------------------------ + */ + + public function toStdClass(); + + /* ------------------------------------------------------------------------------------------ + * Generate a JSON representation of this entity. + * + * @return A JSON string representation for this entity + * ------------------------------------------------------------------------------------------ + */ + + public function toJson(); + + /* ------------------------------------------------------------------------------------------ + * Generate an array containing all SQL statements or fragments required to create + * this entity. Note that some entities such as columns will generate SQL fragments + * and other entities such as triggers may require multiple statements to manage them + * (e.g., DROP TRIGGER, CREATE TRIGGER). Tables with triggers require multiple SQL + * statements to manage them (e.g., CREATE TABLE and CREATE TRIGGER). + * + * @param $includeSchema TRUE to include the schema in the entity names, if + * appropriate. + * + * @return An array comtaining all SQL statements required for creating this item or + * FALSE if there was an error. + * ------------------------------------------------------------------------------------------ + */ + + public function getSql($includeSchema = false); + + /* ------------------------------------------------------------------------------------------ + * @return A string representation for this item. The format of the representation is flexible. + * ------------------------------------------------------------------------------------------ + */ + + public function __toString(); +} // interface iEntity diff --git a/classes/ETL/EtlOverseerOptions.php b/classes/ETL/EtlOverseerOptions.php index de1fc02d4e..55b252641e 100644 --- a/classes/ETL/EtlOverseerOptions.php +++ b/classes/ETL/EtlOverseerOptions.php @@ -13,7 +13,7 @@ use Log; use Exception; use ETL\DataEndpoint\iDataEndpoint; -use ETL\DbEntity\Query; +use ETL\DbModel\Query; class EtlOverseerOptions extends Loggable { diff --git a/classes/ETL/Ingestor/RestIngestor.php b/classes/ETL/Ingestor/RestIngestor.php index 082301c89a..29e8e24c85 100644 --- a/classes/ETL/Ingestor/RestIngestor.php +++ b/classes/ETL/Ingestor/RestIngestor.php @@ -16,7 +16,7 @@ use ETL\DataEndpoint\aRdbmsEndpoint; use ETL\Configuration\EtlConfiguration; use ETL\EtlOverseerOptions; -use ETL\DbEntity\Query; +use ETL\DbModel\Query; use ETL\aOptions; use ETL\iAction; use ETL\Utilities; @@ -256,7 +256,7 @@ protected function performPreExecuteTasks() if ( null !== $this->etlSourceQuery ) { - $sql = $this->etlSourceQuery->getSelectSql(); + $sql = $this->etlSourceQuery->getSql(); if ( null !== $this->variableMap ) { $sql = Utilities::substituteVariables( $sql, @@ -284,7 +284,7 @@ protected function performPreExecuteTasks() $this->processParameters(); - if ( "myisam" == strtolower($this->etlDestinationTable->getEngine()) ) { + if ( "myisam" == strtolower($this->etlDestinationTable->engine) ) { // Disable keys for faster inserts $qualifiedDestTableName = $this->etlDestinationTable->getFullName(); $sqlList = array("ALTER TABLE $qualifiedDestTableName DISABLE KEYS"); @@ -302,7 +302,7 @@ protected function performPreExecuteTasks() protected function performPostExecuteTasks($numRecordsProcessed) { - if ( "myisam" == strtolower($this->etlDestinationTable->getEngine()) ) { + if ( "myisam" == strtolower($this->etlDestinationTable->engine) ) { $qualifiedDestTableName = $this->etlDestinationTable->getFullName(); $sqlList = array("ALTER TABLE $qualifiedDestTableName ENABLE KEYS"); $this->executeSqlList($sqlList, $this->destinationEndpoint); diff --git a/classes/ETL/Ingestor/UpdateIngestor.php b/classes/ETL/Ingestor/UpdateIngestor.php index ef964668b4..ad3e65ca0b 100644 --- a/classes/ETL/Ingestor/UpdateIngestor.php +++ b/classes/ETL/Ingestor/UpdateIngestor.php @@ -169,8 +169,8 @@ public function execute(EtlOverseerOptions $etlOverseerOptions) // The UpdateIngestor does not create the destination table so it must exist. - $tableName = $this->etlDestinationTable->getName(); - $schema = $this->etlDestinationTable->getSchema(); + $tableName = $this->etlDestinationTable->name; + $schema = $this->etlDestinationTable->schema; if ( ! $this->destinationEndpoint->tableExists($tableName, $schema) ) { $msg = "Destination table " . $this->etlDestinationTable->getFullName() . " must exist"; diff --git a/classes/ETL/Ingestor/pdoIngestor.php b/classes/ETL/Ingestor/pdoIngestor.php index 49b550e2c9..fd09c0a914 100644 --- a/classes/ETL/Ingestor/pdoIngestor.php +++ b/classes/ETL/Ingestor/pdoIngestor.php @@ -49,7 +49,7 @@ use ETL\Configuration\EtlConfiguration; use ETL\EtlOverseerOptions; use ETL\Utilities; -use ETL\DbEntity\Query; +use ETL\DbModel\Query; use CCR\DB\MySQLHelper; use PDOException; @@ -199,7 +199,7 @@ public function initialize(EtlOverseerOptions $etlOverseerOptions = null) $this->availableSourceQueryFields = ( null !== $this->etlSourceQuery - ? array_keys($this->etlSourceQuery->getRecords()) + ? array_keys($this->etlSourceQuery->records) : $this->getSqlColumnNames($this->sourceQueryString) ); $this->destinationFieldMappings = $this->getDestinationFields(); @@ -234,7 +234,7 @@ public function initialize(EtlOverseerOptions $etlOverseerOptions = null) $etlTableKey = key($this->etlDestinationTableList); // We only need to parse the SQL if it has been provided as a string, otherwise use: - // array_keys($this->etlSourceQuery->getRecords()); + // array_keys($this->etlSourceQuery->records); $this->destinationFieldMappings[$etlTableKey] = array_combine($this->availableSourceQueryFields, $this->availableSourceQueryFields); @@ -368,7 +368,7 @@ protected function getSourceQueryString() $this->logAndThrowException($msg); } - $sql = $this->etlSourceQuery->getSelectSql(); + $sql = $this->etlSourceQuery->getSql(); if ( null !== $this->variableMap ) { $sql = Utilities::substituteVariables( @@ -478,7 +478,7 @@ protected function performPreExecuteTasks() { foreach ( $this->etlDestinationTableList as $etlTableKey => $etlTable ) { $qualifiedDestTableName = $etlTable->getFullName(); - if ( "myisam" == strtolower($etlTable->getEngine()) ) { + if ( "myisam" == strtolower($etlTable->engine) ) { $disableForeignKeys = true; if ( $this->options->disable_keys ) { $this->logger->info("Disable keys on $qualifiedDestTableName"); @@ -521,7 +521,7 @@ protected function performPostExecuteTasks($numRecordsProcessed) foreach ( $this->etlDestinationTableList as $etlTableKey => $etlTable ) { $qualifiedDestTableName = $etlTable->getFullName(); - if ( "myisam" == strtolower($etlTable->getEngine()) ) { + if ( "myisam" == strtolower($etlTable->engine) ) { $enableForeignKeys = true; if ( $this->options->disable_keys ) { $this->logger->info("Enable keys on $qualifiedDestTableName"); @@ -703,7 +703,7 @@ private function multiDatabaseIngest() } else { $tmpTable = $etlTable->getSchema(true) . "." - . $this->destinationEndpoint->quoteSystemIdentifier("tmp_" . $etlTable->getName() . "_" . time()); + . $this->destinationEndpoint->quoteSystemIdentifier("tmp_" . $etlTable->name . "_" . time()); $destColumns = implode(',', $destColumnList); $updateColumnList = array_map( diff --git a/classes/ETL/Maintenance/ManageTables.php b/classes/ETL/Maintenance/ManageTables.php index 11b9d9bb05..f3c0a50c0a 100644 --- a/classes/ETL/Maintenance/ManageTables.php +++ b/classes/ETL/Maintenance/ManageTables.php @@ -15,7 +15,7 @@ use ETL\Configuration\EtlConfiguration; use ETL\EtlOverseerOptions; -use ETL\DbEntity\Table; +use ETL\DbModel\Table; use ETL\aOptions; use ETL\iAction; use ETL\aRdbmsDestinationAction; @@ -103,8 +103,8 @@ protected function createDestinationTableObjects() $this->destinationEndpoint->getSystemQuoteChar(), $this->logger ); - $etlTable->setSchema($this->destinationEndpoint->getSchema()); - $this->etlDestinationTableList[$etlTable->getName()] = $etlTable; + $etlTable->schema = $this->destinationEndpoint->getSchema(); + $this->etlDestinationTableList[$etlTable->name] = $etlTable; } } // createDestinationTableObjects() diff --git a/classes/ETL/Maintenance/VerifyDatabase.php b/classes/ETL/Maintenance/VerifyDatabase.php index 09422ae157..2a5adc4662 100644 --- a/classes/ETL/Maintenance/VerifyDatabase.php +++ b/classes/ETL/Maintenance/VerifyDatabase.php @@ -16,7 +16,7 @@ use ETL\iAction; use ETL\aAction; use ETL\DataEndpoint\iRdbmsEndpoint; -use ETL\DbEntity\Query; +use ETL\DbModel\Query; use \PDOException; use ETL\Utilities; use \Log; @@ -140,10 +140,10 @@ public function initialize(EtlOverseerOptions $etlOverseerOptions = null) $this->sourceEndpoint->getSystemQuoteChar(), $this->logger ); - $this->queryColumnNames = array_keys($sourceQuery->getRecords()); + $this->queryColumnNames = array_keys($sourceQuery->records); $this->setOverseerRestrictionOverrides(); $this->getEtlOverseerOptions()->applyOverseerRestrictions($sourceQuery, $this->sourceEndpoint, $this); - $this->sqlQueryString = $sourceQuery->getSelectSql(); + $this->sqlQueryString = $sourceQuery->getSql(); $this->sqlQueryString = Utilities::substituteVariables( $this->sqlQueryString, $this->variableMap, diff --git a/classes/ETL/Utilities.php b/classes/ETL/Utilities.php index 7ab8dca5a5..b00b8d9b28 100644 --- a/classes/ETL/Utilities.php +++ b/classes/ETL/Utilities.php @@ -68,6 +68,10 @@ public static function substituteVariables( array $substitutionDetails = null ) { + if ( null === $string ) { + return $string; + } + $exceptionForUnusedVariables = ( null !== $logger ); $trackDetails = ( null !== $substitutionDetails ); diff --git a/classes/ETL/aRdbmsDestinationAction.php b/classes/ETL/aRdbmsDestinationAction.php index 02f348cb8c..0188582174 100644 --- a/classes/ETL/aRdbmsDestinationAction.php +++ b/classes/ETL/aRdbmsDestinationAction.php @@ -23,7 +23,7 @@ use ETL\DataEndpoint\iDataEndpoint; use ETL\DataEndpoint\iRdbmsEndpoint; use ETL\aOptions; -use ETL\DbEntity\Table; +use ETL\DbModel\Table; use PHPSQLParser\PHPSQLParser; @@ -155,10 +155,10 @@ protected function createDestinationTableObjects() ); $this->logger->debug( "Created ETL destination table object for table definition key '" - . $etlTable->getName() + . $etlTable->name . "'" ); - $etlTable->setSchema($this->destinationEndpoint->getSchema()); + $etlTable->schema = $this->destinationEndpoint->getSchema(); $tableName = $etlTable->getFullName(); if ( ! is_string($tableName) || empty($tableName) ) @@ -167,7 +167,7 @@ protected function createDestinationTableObjects() $this->logAndThrowException($msg); } - $this->etlDestinationTableList[$etlTable->getName()] = $etlTable; + $this->etlDestinationTableList[$etlTable->name] = $etlTable; } catch (Exception $e) { $this->logAndThrowException($e->getMessage() . " in file '" . $this->definitionFile . "'"); } @@ -224,7 +224,7 @@ protected function performTruncateDestinationTasks() try { - if ( false === $this->destinationEndpoint->tableExists($etlTable->getName(), $etlTable->getSchema()) ) { + if ( false === $this->destinationEndpoint->tableExists($etlTable->name, $etlTable->schema) ) { $this->logger->info("Table does not exist: '$tableName', skipping."); continue; } @@ -378,7 +378,7 @@ public function verifySqlColumns($sql, Table $table) $missingColumnNames = array_diff($sqlColumnNames, $tableColumnNames); if ( 0 != count($missingColumnNames) ) { - $msg = "The following columns from the SQL SELECT were not found in table definition for '{$table->getName()}': " . + $msg = "The following columns from the SQL SELECT were not found in table definition for '{$table->name}': " . implode(", ", $missingColumnNames); $this->logAndThrowException($msg); } @@ -406,20 +406,15 @@ public function manageTable(Table $table, iDataEndpoint $endpoint) { // Check for an existing table with the same name - $existingTable = Table::discover( - $table->getName(), - $endpoint, - $endpoint->getSystemQuoteChar(), - $this->logger - ); + $existingTable = new Table(null, $endpoint->getSystemQuoteChar(), $this->logger); // If no table with that name exists, create it. Otherwise check for differences and apply them. - if ( false === $existingTable ) { + if ( false === $existingTable->discover($table->name, $endpoint) ) { $this->logger->notice("Table " . $table->getFullName() . " does not exist, creating."); - $sqlList = $table->getCreateSql(); + $sqlList = $table->getSql(); foreach ( $sqlList as $sql ) { $this->logger->debug("Create table SQL " . $endpoint . ":\n$sql"); diff --git a/composer.lock b/composer.lock index 1e121c1165..94e7855aad 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", "This file is @generated automatically" ], - "content-hash": "ceeef77ecbe7a0be2763939f54b3ba06", + "content-hash": "d0538f717752f90e1898bee194ced712", "packages": [ { "name": "apache/commons-beanutils", @@ -2695,12 +2695,12 @@ "source": { "type": "git", "url": "https://github.com/ubccr/xdmod-test-artifacts.git", - "reference": "5cd5bde2b2041e9d71532c3e5e9a8fbddeada679" + "reference": "173dcefc611ba7eabd1aa9aa3c400372e7f9aaf7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/ubccr/xdmod-test-artifacts/zipball/5cd5bde2b2041e9d71532c3e5e9a8fbddeada679", - "reference": "5cd5bde2b2041e9d71532c3e5e9a8fbddeada679", + "url": "https://api.github.com/repos/ubccr/xdmod-test-artifacts/zipball/f4a08a5244cf41d212e7655b4c87276fb04f2494", + "reference": "173dcefc611ba7eabd1aa9aa3c400372e7f9aaf7", "shasum": "" }, "type": "library", @@ -2709,7 +2709,7 @@ "source": "https://github.com/ubccr/xdmod-test-artifacts/tree/master", "issues": "https://github.com/ubccr/xdmod-test-artifacts/issues" }, - "time": "2017-04-21T18:47:07+00:00" + "time": "2017-04-26 17:59:22" } ], "aliases": [], diff --git a/configuration/etl/etl_tables.d/jobs/jobfact_hpc_aggregation.json b/configuration/etl/etl_tables.d/jobs/jobfact_hpc_aggregation.json index a85f077598..8b6adfea2d 100644 --- a/configuration/etl/etl_tables.d/jobs/jobfact_hpc_aggregation.json +++ b/configuration/etl/etl_tables.d/jobs/jobfact_hpc_aggregation.json @@ -3,6 +3,7 @@ "table_definition": { "name": "jobfact_by_", + "table_prefix": "jobfact_by_", "engine": "MyISAM", "comment": "Jobfacts aggregated by ${AGGREGATION_UNIT}.", "columns": [ diff --git a/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/EtlConfigurationTest.php b/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/EtlConfigurationTest.php index 02bea1c856..120a16fc23 100644 --- a/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/EtlConfigurationTest.php +++ b/open_xdmod/modules/xdmod/tests/lib/ETL/Configuration/EtlConfigurationTest.php @@ -26,7 +26,7 @@ public function testConfiguration() // The test files need to be in the same location that the expected results were // generated from or paths stored in the expected result will not match! - mkdir(self::TMPDIR . '/etl.d', 0755, true); + @mkdir(self::TMPDIR . '/etl.d', 0755, true); copy(self::TEST_ARTIFACT_INPUT_PATH . '/xdmod_etl_config.json', self::TMPDIR . '/xdmod_etl_config.json'); copy(self::TEST_ARTIFACT_INPUT_PATH . '/etl.d/maintenance.json', self::TMPDIR . '/etl.d/maintenance.json'); copy(self::TEST_ARTIFACT_INPUT_PATH . '/etl.d/jobs_cloud.json', self::TMPDIR . '/etl.d/jobs_cloud.json'); diff --git a/open_xdmod/modules/xdmod/tests/lib/ETL/DbModel/DbModelTest.php b/open_xdmod/modules/xdmod/tests/lib/ETL/DbModel/DbModelTest.php new file mode 100644 index 0000000000..e68546309d --- /dev/null +++ b/open_xdmod/modules/xdmod/tests/lib/ETL/DbModel/DbModelTest.php @@ -0,0 +1,396 @@ + + * @date 2017-04-21 + * ------------------------------------------------------------------------------------------ + */ + +namespace UnitTesting\ETL\Configuration; + +use CCR\Log; +use ETL\Utilities; +use ETL\DbModel\Table; +use ETL\DbModel\AggregationTable; +use ETL\DbModel\Query; +use ETL\DbModel\Column; +use ETL\DbModel\Index; +use ETL\DbModel\Trigger; +use ETL\Configuration\EtlConfiguration; + +class DbModelTest extends \PHPUnit_Framework_TestCase +{ + const TEST_ARTIFACT_INPUT_PATH = "../../../../vendor/ubccr/xdmod-test-artifacts/xdmod/etlv2/dbmodel/input"; + const TEST_ARTIFACT_OUTPUT_PATH = "../../../../vendor/ubccr/xdmod-test-artifacts/xdmod/etlv2/dbmodel/output"; + private $logger = null; + + public function __construct() + { + // Set up a logger so we can get warnings and error messages from the ETL + // infrastructure + $conf = array( + 'db' => false, + 'mail' => false, + 'consoleLogLevel' => Log::WARNING + ); + $this->logger = Log::factory('PHPUnit', $conf); + } + + /** + * Test creating a table from a JSON file and feeding the generated JSON back to generate + * the same table. + */ + + public function testParseJsonFile() + { + // Instantiate the reference table + $config = self::TEST_ARTIFACT_INPUT_PATH . '/table_def.json'; + $table = new Table($config, '`', $this->logger); + $table->verify(); + + // Verify SQL generated from JSON + $generated = $table->getSql(); + $generated = array_shift($generated); + $expected = trim(file_get_contents(self::TEST_ARTIFACT_OUTPUT_PATH . '/table_def.sql')); + $this->assertEquals($expected, $generated); + + // Run the generated JSON through and verify the generated SQL again. + $newTable = new Table(json_decode($table->toJson()), '`', $this->logger); + $generated = $newTable->getSql(); + $generated = array_shift($generated); + $this->assertEquals($expected, $generated); + } + + /** + * Test generating SQL with and without a table schema. + */ + + public function testTableSchema() + { + $config = (object) array( + 'name' => "table_no_schema", + 'columns' => array( (object) array( + 'name' => 'column1', + 'type' => 'int(11)', + 'nullable' => true, + 'default' => 0, + 'comment' => 'This is my comment' + )) + ); + + $table = new Table($config, '`', $this->logger); + $table->schema = "my_schema"; + $table->verify(); + + // SQL with no schema + $generated = $table->getSql(false); + $generated = array_shift($generated); + $expected = "CREATE TABLE IF NOT EXISTS `table_no_schema` ( + `column1` int(11) NULL DEFAULT 0 COMMENT 'This is my comment' +);"; + $this->assertEquals($expected, $generated); + + // SQL with schema + $generated = $table->getSql(); + $generated = array_shift($generated); + $expected = "CREATE TABLE IF NOT EXISTS `my_schema`.`table_no_schema` ( + `column1` int(11) NULL DEFAULT 0 COMMENT 'This is my comment' +);"; + $this->assertEquals($expected, $generated); + } + + /** + * Test table verification error + * + * @expectedException Exception + */ + + public function testTableVerificationError() + { + $config = (object) array( + 'name' => "verification_error", + 'columns' => array( (object) array( + 'name' => 'column1', + 'type' => 'int(11)', + 'nullable' => true, + 'default' => 0, + 'comment' => 'This is my comment' + )), + 'indexes' => array( (object) array( + 'columns' => array('column1', 'missing_column') + )) + ); + + $table = new Table($config); // No logger here + $table->verify(); + } + + /** + * Verify creating table elements manually. + */ + + public function testCreateSql() + { + $config = (object) array( + 'name' => 'column1', + 'type' => 'int(11)', + 'nullable' => true, + 'default' => 0, + 'comment' => 'This is my comment' + ); + + $obj = new Column($config, '`', $this->logger); + $generated = $obj->getSql(); + $expected = "`column1` int(11) NULL DEFAULT 0 COMMENT 'This is my comment'"; + $this->assertEquals($expected, $generated); + + $config = (object) array( + 'columns' => array('col1', 'col2') + ); + + // Test with a system quote character + $obj = new Index($config, '`', $this->logger); + $generated = $obj->getSql(); + $expected = "INDEX `index_col1_col2` (`col1`, `col2`)"; + $this->assertEquals($expected, $generated); + + // Test with no system quote character + $obj = new Index($config, null, $this->logger); + $generated = $obj->getSql(); + $expected = "INDEX index_col1_col2 (col1, col2)"; + $this->assertEquals($expected, $generated); + + $config = (object) array( + 'name' => 'before_ins', + 'time' => 'before', + 'event' => 'insert', + 'table' => 'jobfact', + 'body' => 'BEGIN DELETE FROM jobfactstatus WHERE job_id = NEW.job_id; END' + ); + + $obj = new Trigger($config, '`', $this->logger); + $generated = $obj->getSql(); + $expected = + "CREATE TRIGGER `before_ins` before insert ON `jobfact` FOR EACH ROW" + . PHP_EOL + . " BEGIN DELETE FROM jobfactstatus WHERE job_id = NEW.job_id; END"; + $this->assertEquals($expected, $generated); + + } + + /** + * Test comparing 2 tables and the ALTER TABLE statement needed to go from one to the other. + * Also manually add elements and verify the ALTER TABLE statement generated. + */ + + public function testAlterTable() + { + // Instantiate the reference table + $config = self::TEST_ARTIFACT_INPUT_PATH . '/table_def.json'; + $currentTable = new Table($config, '`', $this->logger); + $currentTable->verify(); + $config = self::TEST_ARTIFACT_INPUT_PATH . '/table_def_2.json'; + $destTable = new Table($config, '`', $this->logger); + $destTable->verify(); + + $generated = $currentTable->getAlterSql($destTable); + $generated = array_shift($generated); + $expected = trim(file_get_contents(self::TEST_ARTIFACT_OUTPUT_PATH . '/alter_table.sql')); + // Assert that there is no alter sql statement. + $this->assertEquals($expected, $generated); + + // Alter the table by manually adding a column, index, and trigger. + + $config = (object) array( + 'name' => 'new_column', + 'type' => 'boolean', + 'nullable' => false, + 'default' => 0 + ); + $destTable->addColumn($config); + + $config = (object) array( + 'columns' => array('new_column') + ); + $destTable->addIndex($config); + + $config = (object) array( + 'name' => 'before_ins', + 'time' => 'before', + 'event' => 'insert', + 'table' => 'jobfact', + 'body' => 'BEGIN DELETE FROM jobfactstatus WHERE job_id = NEW.job_id; END' + ); + $destTable->addTrigger($config); + + // The getSql() and getSql() methods return an array containing distinct SQL + // statements. + $generated = $currentTable->getAlterSql($destTable); + $alterTable = array_shift($generated); + $trigger = array_shift($generated); + $generated = $alterTable . PHP_EOL . $trigger; + $expected = trim(file_get_contents(self::TEST_ARTIFACT_OUTPUT_PATH . '/alter_table_manually.sql')); + $this->assertEquals($expected, $generated); + } + + /** + * Test removing all elements from the table + */ + + public function testDeleteTableElements() + { + // Instantiate the reference table + $config = self::TEST_ARTIFACT_INPUT_PATH . '/table_def.json'; + $table = new Table($config, '`', $this->logger); + $table->verify(); + + $table->resetPropertyValues(); + $this->assertFalse($table->getSql()); + } + + /** + * Test the query object including variable substitution + */ + + public function testQuery() + { + $config = json_decode(file_get_contents(self::TEST_ARTIFACT_INPUT_PATH . '/resource_allocations.json')); + $query = new Query($config->source_query, '"', $this->logger); + $generated = $query->getSql(); + + // Process variables present in the SQL + $variableMap = array( + 'TIMEZONE' => 'America/New_York', + 'SOURCE_SCHEMA' => 'xras' + ); + $generated = Utilities::substituteVariables( + $generated, + $variableMap, + $query, + "Undefined macros found in source query" + ); + + $expected = trim(file_get_contents(self::TEST_ARTIFACT_OUTPUT_PATH . '/resource_allocations.sql')); + $this->assertEquals($expected, $generated); + } + + /** + * Test generating an AggregationTable + */ + + public function testAggregationTable() + { + $aggregationUnit = 'quarter'; + $file = self::TEST_ARTIFACT_INPUT_PATH . '/resourceallocationfact_by.aggregation.json'; + + $config = json_decode(file_get_contents($file)); + $table = new AggregationTable($config, '`', $this->logger); + $table->aggregation_unit = $aggregationUnit; + $generated = $table->getSql(); + $generated = array_shift($generated); + + // Process variables present in the SQL + $variableMap = array( + 'AGGREGATION_UNIT' => $aggregationUnit, + 'SOURCE_SCHEMA' => 'xras' + ); + $generated = Utilities::substituteVariables( + $generated, + $variableMap, + $table, + "Undefined macros found in source query" + ); + + $file = self::TEST_ARTIFACT_OUTPUT_PATH . '/resourceallocationfact_by.aggregation.sql'; + $expected = trim(file_get_contents($file)); + $this->assertEquals($expected, $generated); + } + + /** + * Test generating SQL from an aggregation source query + */ + + public function testAggregationTableQuery() + { + $aggregationUnit = 'quarter'; + $file = self::TEST_ARTIFACT_INPUT_PATH . '/resourceallocationfact_by.aggregation.json'; + + $config = json_decode(file_get_contents($file)); + $table = new AggregationTable($config, '`', $this->logger); + $table->aggregation_unit = $aggregationUnit; + $generated = $table->query->getSql(); + + // Process variables present in the SQL + $variableMap = array( + 'AGGREGATION_UNIT' => $aggregationUnit, + 'SOURCE_SCHEMA' => 'modw_ra', + 'UTILITY_SCHEMA' => 'modw', + ':PERIOD_ID' => ':period_id', + ':YEAR_VALUE' => ':year_value', + ':PERIOD_VALUE' => ':period_value', + ':PERIOD_START_TS' => ':period_start_ts', + ':PERIOD_END_TS' => ':period_end_ts' + ); + $generated = Utilities::substituteVariables( + $generated, + $variableMap, + $table, + "Undefined macros found in source query" + ); + + $file = self::TEST_ARTIFACT_OUTPUT_PATH . '/resourceallocationfact_by.query.sql'; + $expected = trim(file_get_contents($file)); + $this->assertEquals($expected, $generated); + } + + /** + * Test generating a stdClass from an object + */ + + public function testGenerateQueryStdClass() + { + // Generate a query + $config = json_decode(file_get_contents(self::TEST_ARTIFACT_INPUT_PATH . '/resource_allocations.json')); + $query = new Query($config->source_query, '"', $this->logger); + + // Generate the stdclass and pass it back to generate the same query + $obj = $query->toStdClass(); + $newQuery = new Query($obj, '"', $this->logger); + $generated = $newQuery->getSql(); + + $variableMap = array( + 'TIMEZONE' => 'America/New_York', + 'SOURCE_SCHEMA' => 'xras' + ); + $generated = Utilities::substituteVariables( + $generated, + $variableMap, + $newQuery, + "Undefined macros found in source query" + ); + $expected = trim(file_get_contents(self::TEST_ARTIFACT_OUTPUT_PATH . '/resource_allocations.sql')); + $this->assertEquals($expected, $generated); + } + + /** + * Test generating a stdClass from an table + */ + + public function testGenerateTableStdClass() + { + // Instantiate the reference table + $config = self::TEST_ARTIFACT_INPUT_PATH . '/table_def.json'; + $table = new Table($config, '`', $this->logger); + $table->verify(); + + // Generate the stdclass and pass it back to generate the same table + $obj = $table->toStdClass(); + $newTable = new Table($obj, '`', $this->logger); + $generated = $newTable->getSql(); + $generated = array_shift($generated); + + $expected = trim(file_get_contents(self::TEST_ARTIFACT_OUTPUT_PATH . '/table_def.sql')); + $this->assertEquals($expected, $generated); + } +} // class ConfigurationTest diff --git a/open_xdmod/modules/xdmod/tests/phpunit.xml.dist b/open_xdmod/modules/xdmod/tests/phpunit.xml.dist index 578d81b697..97e943f669 100644 --- a/open_xdmod/modules/xdmod/tests/phpunit.xml.dist +++ b/open_xdmod/modules/xdmod/tests/phpunit.xml.dist @@ -18,4 +18,11 @@ stopOnSkipped="false" testSuiteLoaderClass="PHPUnit_Runner_StandardTestSuiteLoader" verbose="true"> + + + /usr/share/pear + /usr/share/php + ../../../../vendor + + diff --git a/tools/etl/etl_table_manager.php b/tools/etl/etl_table_manager.php index 6f4099ff1b..5d4b2063f1 100755 --- a/tools/etl/etl_table_manager.php +++ b/tools/etl/etl_table_manager.php @@ -1,11 +1,11 @@ #!/usr/bin/env php - - */ + + */ require __DIR__ . '/../../configuration/linker.php'; restore_exception_handler(); @@ -13,8 +13,8 @@ use \Exception; use CCR\Log; use ETL\Configuration\EtlConfiguration; -use ETL\DbEntity\Table; -use ETL\DbEntity\AggregationTable; +use ETL\DbModel\Table; +use ETL\DbModel\AggregationTable; $supportedFormats = array("json", "sql"); @@ -23,27 +23,27 @@ $scriptOptions = array( // ETL configuration file - 'config-file' => NULL, + 'config-file' => null, // Endpoint (defined in the ETL config) to use when querying tables 'endpoint' => "utility", // Table to use in discovery mode, needed for alter statement - 'discover-table' => NULL, + 'discover-table' => null, // TRUE to include the schema name in tables and triggers - 'include-schema' => FALSE, + 'include-schema' => false, // Operation to perform - 'operation' => NULL, + 'operation' => null, // Output file - 'output-file' => NULL, + 'output-file' => null, // Output format (json or sql) 'output-format' => 'json', // Succinct or verbose mode - 'succinct-mode' => FALSE, + 'succinct-mode' => false, // Table definition file - 'table-config' => NULL, + 'table-config' => null, // Key name that the table definition will be included under - 'table-key' => NULL, + 'table-key' => null, 'verbosity' => Log::NOTICE - ); +); // ========================================================================================== // Process command line arguments @@ -61,95 +61,97 @@ 't:' => 'table-config:', 'v:' => 'verbosity:', 'x:' => 'output-format:' - ); +); $args = getopt(implode('', array_keys($options)), $options); foreach ($args as $arg => $value) { switch ($arg) { - case 'c': - case 'config-file': - $scriptOptions['config-file'] = $value; - break; - - case 'd': - case 'discover-table': - $scriptOptions['discover-table'] = $value; - break; - - case 'e': - case 'endpoint': - $scriptOptions['endpoint'] = $value; - break; + case 'c': + case 'config-file': + $scriptOptions['config-file'] = $value; + break; - case 'f': - case 'output-file': - $scriptOptions['output-file'] = $value; - break; + case 'd': + case 'discover-table': + $scriptOptions['discover-table'] = $value; + break; - case 'i': - case 'include-schema': - $scriptOptions['include-schema'] = TRUE; - break; + case 'e': + case 'endpoint': + $scriptOptions['endpoint'] = $value; + break; - case 'k': - case 'table-key': - $scriptOptions['table-key'] = $value; - break; + case 'f': + case 'output-file': + $scriptOptions['output-file'] = $value; + break; - case 'o': - case 'operation': - $scriptOptions['operation'] = $value; - break; + case 'i': + case 'include-schema': + $scriptOptions['include-schema'] = true; + break; - case 's': - case 'succinct': - $scriptOptions['succinct-mode'] = TRUE; - break; + case 'k': + case 'table-key': + $scriptOptions['table-key'] = $value; + break; - case 't': - case 'table-config': - $scriptOptions['table-config'] = $value; - break; + case 'o': + case 'operation': + $scriptOptions['operation'] = $value; + break; - case 'v': - case 'verbosity': - switch ( $value ) { - case 'debug': - $scriptOptions['verbosity'] = Log::DEBUG; + case 's': + case 'succinct': + $scriptOptions['succinct-mode'] = true; break; - case 'info': - $scriptOptions['verbosity'] = Log::INFO; + + case 't': + case 'table-config': + $scriptOptions['table-config'] = $value; break; - case 'notice': - $scriptOptions['verbosity'] = Log::NOTICE; + + case 'v': + case 'verbosity': + switch ( $value ) { + case 'debug': + $scriptOptions['verbosity'] = Log::DEBUG; + break; + case 'info': + $scriptOptions['verbosity'] = Log::INFO; + break; + case 'notice': + $scriptOptions['verbosity'] = Log::NOTICE; + break; + case 'warning': + $scriptOptions['verbosity'] = Log::WARNING; + break; + case 'quiet': + $scriptOptions['verbosity'] = Log::EMERG; + break; + default: + break; + } // switch ( $value ) break; - case 'warning': - $scriptOptions['verbosity'] = Log::WARNING; + + case 'x': + case 'output-format': + $value = strtolower($value); + if ( ! in_array($value, $supportedFormats) ) { + usage_and_exit("Unsupported output format"); + } + $scriptOptions['output-format'] = $value; break; - case 'quiet': - $scriptOptions['verbosity'] = Log::EMERG; + + case 'h': + case 'help': + usage_and_exit(); break; + default: break; - } // switch ( $value ) - break; - - case 'x': - case 'output-format': - $value = strtolower($value); - if ( ! in_array($value, $supportedFormats) ) usage_and_exit("Unsupported output format"); - $scriptOptions['output-format'] = $value; - break; - - case 'h': - case 'help': - usage_and_exit(); - break; - - default: - break; } } // foreach ($args as $arg => $value) @@ -158,17 +160,19 @@ $conf = array( 'emailSubject' => gethostname() . ': XDMOD: Data Warehouse: Federated ETL Log', - 'mail' => FALSE - ); + 'mail' => false +); -if ( NULL !== $scriptOptions['verbosity'] ) $conf['consoleLogLevel'] = $scriptOptions['verbosity']; +if ( null !== $scriptOptions['verbosity'] ) { + $conf['consoleLogLevel'] = $scriptOptions['verbosity']; +} $logger = Log::factory('DWI', $conf); -if ( NULL === $scriptOptions['config-file'] || - NULL === $scriptOptions['operation'] ) { +if ( null === $scriptOptions['config-file'] || + null === $scriptOptions['operation'] ) { usage_and_exit("Must supply config file and operation"); -} else if ( ! is_file($scriptOptions['config-file']) ) { +} elseif ( ! is_file($scriptOptions['config-file']) ) { usage_and_exit("Config file not found: '" . $scriptOptions['config-file'] . "'"); } @@ -181,22 +185,22 @@ $etlConfig->initialize(); } catch ( Exception $e ) { exit($e->getMessage() . "\n"); - } +} // ------------------------------------------------------------------------------------------ // Verify the requested endpoint exists -$dataEndpoint = NULL; -if ( FALSE === ($dataEndpoint = $etlConfig->getGlobalEndpoint($scriptOptions['endpoint'])) ) { +$dataEndpoint = null; +if ( false === ($dataEndpoint = $etlConfig->getGlobalEndpoint($scriptOptions['endpoint'])) ) { $msg = "Global endpoint '{$scriptOptions['endpoint']}' not defined, cannot query database for resource code mapping"; throw new Exception($msg); } // ------------------------------------------------------------------------------------------ -$parsedTable = NULL; +$parsedTable = null; -if ( NULL !== $scriptOptions['table-config'] ) { +if ( null !== $scriptOptions['table-config'] ) { try { // $parsedTable = new AggregationTable($scriptOptions['table-config']); $parsedTable = new Table($scriptOptions['table-config']); @@ -209,12 +213,12 @@ } } -$discoveredTable = NULL; +$discoveredTable = null; -if ( NULL !== $scriptOptions['discover-table'] ) { +if ( null !== $scriptOptions['discover-table'] ) { try { $discoveredTable = Table::discover($scriptOptions['discover-table'], $dataEndpoint); - if ( FALSE === $discoveredTable ) { + if ( false === $discoveredTable ) { $msg = "Table '" . $scriptOptions['discover-table'] . "' not found using endpoint $dataEndpoint\n"; exit($msg); } @@ -227,65 +231,71 @@ // Perform the requested operation -$outputStr = NULL; +$outputStr = null; switch ( $scriptOptions['operation'] ) { -case 'dump-discovered': - if ( NULL !== $discoveredTable ) { - $outputStr = ""; - if ( "json" == $scriptOptions['output-format'] ) { - $obj = $discoveredTable->toJsonObj($scriptOptions['succinct-mode'], $scriptOptions['include-schema']); - if ( NULL !== $scriptOptions['table-key'] ) { - $tableKey = $scriptOptions['table-key']; - $retval = new stdClass; - $retval->$tableKey = $obj; - $obj = $retval; + case 'dump-discovered': + if ( null !== $discoveredTable ) { + $outputStr = ""; + if ( "json" == $scriptOptions['output-format'] ) { + $obj = $discoveredTable->toJsonObj($scriptOptions['succinct-mode'], $scriptOptions['include-schema']); + if ( null !== $scriptOptions['table-key'] ) { + $tableKey = $scriptOptions['table-key']; + $retval = new stdClass; + $retval->$tableKey = $obj; + $obj = $retval; + } + $outputStr = json_encode($obj); + } else { + $outputStr = "DELIMITER ;;\n" . + implode("\n;;\n", $discoveredTable->getCreateSql($scriptOptions['include-schema'])) . + "\n;;"; } - $outputStr = json_encode($obj); - } else { - $outputStr = "DELIMITER ;;\n" . - implode("\n;;\n", $discoveredTable->getCreateSql($scriptOptions['include-schema'])) . - "\n;;"; } - } - break; - -case 'dump-parsed': - if ( NULL !== $parsedTable ) { - if ( "json" == $scriptOptions['output-format'] ) { - $obj = $parsedTable->toJsonObj($scriptOptions['succinct-mode'], $scriptOptions['include-schema']); - if ( NULL !== $scriptOptions['table-key'] ) { - $tableKey = $scriptOptions['table-key']; - $retval = new stdClass; - $retval->$tableKey = $obj; - $obj = $retval; + break; + + case 'dump-parsed': + if ( null !== $parsedTable ) { + if ( "json" == $scriptOptions['output-format'] ) { + $obj = $parsedTable->toJsonObj($scriptOptions['succinct-mode'], $scriptOptions['include-schema']); + if ( null !== $scriptOptions['table-key'] ) { + $tableKey = $scriptOptions['table-key']; + $retval = new stdClass; + $retval->$tableKey = $obj; + $obj = $retval; + } + $outputStr = json_encode($obj); + } else { + $outputStr = "DELIMITER ;;\n" . + implode("\n;;\n", $parsedTable->getCreateSql($scriptOptions['include-schema'])) . + "\n;;"; } - $outputStr = json_encode($obj); - } else { - $outputStr = "DELIMITER ;;\n" . - implode("\n;;\n", $parsedTable->getCreateSql($scriptOptions['include-schema'])) . - "\n;;"; } - } - break; + break; -case 'dump-alter': - if ( NULL !== $discoveredTable && NULL !== $parsedTable ) { - if ( "json" == $scriptOptions['output-format'] ) usage_and_exit("JSON format not supported for ALTER TABLE"); - $alterSqlList = $discoveredTable->getAlterSql($parsedTable, $scriptOptions['include-schema']); - if ( $alterSqlList ) $outputStr = "DELIMITER ;;\n" . implode("\n;;\n", $alterSqlList) . "\n;;"; - } - break; + case 'dump-alter': + if ( null !== $discoveredTable && null !== $parsedTable ) { + if ( "json" == $scriptOptions['output-format'] ) { + usage_and_exit("JSON format not supported for ALTER TABLE"); + } + $alterSqlList = $discoveredTable->getAlterSql($parsedTable, $scriptOptions['include-schema']); + if ( $alterSqlList ) { + $outputStr = "DELIMITER ;;\n" . implode("\n;;\n", $alterSqlList) . "\n;;"; + } + } + break; -default: - usage_and_exit("Unknown operation"); - break; + default: + usage_and_exit("Unknown operation"); + break; } -if ( NULL === $outputStr ) exit(0); +if ( null === $outputStr ) { + exit(0); +} -if ( NULL !== $scriptOptions['output-file'] ) { +if ( null !== $scriptOptions['output-file'] ) { file_put_contents($scriptOptions['output-file'], "$outputStr\n"); } else { fwrite(STDOUT, "$outputStr\n"); @@ -353,7 +363,7 @@ function usage_and_exit($msg = null) Output format ($availablelFormats) EOMSG - ); + ); exit(1); }