Show raw data jobs

classes/DataWarehouse/Query/Jobs/JobDataset.php

@@ -0,0 +1,127 @@
+<?php
+/**
+* @author Joe White
+* @date 2015-03-25
+*/
+namespace DataWarehouse\Query\Jobs;
+
+use \DataWarehouse\Query\Model\Table;
+use \DataWarehouse\Query\Model\TableField;
+use \DataWarehouse\Query\Model\FormulaField;
+use \DataWarehouse\Query\Model\WhereCondition;
+use \DataWarehouse\Query\Model\Schema;
+
+/*
+ * @see DataWarehouse::Query::RawQuery
+ */
+class JobDataset extends \DataWarehouse\Query\RawQuery
+{
+    private $documentation = array();
+
+    public function __construct(
+        array $parameters,
+        $stat = "all"
+    ) {
+
+        parent::__construct('Jobs', 'modw_aggregates', 'jobfact_by_day', array());
+
+        $config = \Xdmod\Config::factory();
+
+        $dataTable = $this->getDataTable();
+        $joblistTable = new Table($dataTable->getSchema(), $dataTable->getName() . "_joblist", "jl");
+        $factTable = new Table(new Schema('modw'), 'job_tasks', 'jt');
+
+        $this->addTable($joblistTable );
+        $this->addTable($factTable );
+
+        $this->addWhereCondition(new WhereCondition(
+            new TableField($joblistTable, "agg_id"),
+            "=",
+            new TableField($dataTable, "id")
+        ));
+        $this->addWhereCondition(new WhereCondition(
+            new TableField($joblistTable, "jobid"),
+            "=",
+            new TableField($factTable, "job_id")
+        ));
+
+        if (isset($parameters['primary_key'])) {
+            $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'job_id'), "=", $parameters['primary_key']));
+        } else {
+            $matches = array();
+            if (preg_match('/^(\d+)(?:[\[_](\d+)\]?)?$/', $parameters['job_identifier'], $matches)) {
+                $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'resource_id'), '=', $parameters['resource_id']));
+                if (isset($matches[2])) {
+                    $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'local_jobid'), '=', $matches[1]));
+                    $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'local_job_array_index'), '=', $matches[2]));
+                } else {
+                    $this->addPdoWhereCondition(new WhereCondition(new TableField($factTable, 'local_job_id_raw'), '=', $matches[1]));
+                }
+            } else {
+                throw new \Exception('invalid query parameters');
+            }
+        }
+
+        if ($stat == "accounting") {
+            $i = 0;
+            foreach ($config['rawstatistics']['modw.job_tasks'] as $sdata) {
+                $sfield = $sdata['key'];
+                if ($sdata['dtype'] == 'accounting') {
+                    $this->addField(new TableField($factTable, $sfield));
+                    $this->documentation[$sfield] = $sdata;
+                } elseif ($sdata['dtype'] == 'foreignkey') {
+                    if (isset($sdata['join'])) {
+                        $info = $sdata['join'];
+                        $i += 1;
+                        $tmptable = new Table(new Schema($info['schema']), $info['table'], "ft$i");
+                        $this->addTable($tmptable);
+                        $this->addWhereCondition(new WhereCondition(new TableField($factTable, $sfield), '=', new TableField($tmptable, "id")));
+                        $fcol = isset($info['column']) ? $info['column'] : 'name';
+                        $this->addField(new TableField($tmptable, $fcol, $sdata['name']));
+
+                        $this->documentation[ $sdata['name'] ] = $sdata;
+                    }
+                }
+            }
+            $rf = new Table(new Schema('modw'), 'resourcefact', 'rf');
+            $this->addTable($rf);
+            $this->addWhereCondition(new WhereCondition(new TableField($factTable, 'resource_id'), '=', new TableField($rf, 'id')));
+            $this->addField(new TableField($rf, 'timezone'));
+            $this->documentation['timezone'] = array(
+                "name" => "Timezone",
+                "documentation" => "The timezone of the resource.",
+                "group" => "Administration",
+                'visibility' => 'public',
+                "per" => "resource");
+        }
+        else
+        {
+            $this->addField(new TableField($factTable, "job_id", "jobid"));
+            $this->addField(new TableField($factTable, "local_jobid", "local_job_id"));
+
+            $rt = new Table(new Schema("modw"), "resourcefact", "rf");
+            $this->joinTo($rt, "task_resource_id", "code", "resource");
+
+            $pt = new Table(new Schema('modw'), 'person', 'p');
+            $this->joinTo($pt, "person_id", "long_name", "name");
+
+            $st = new Table(new Schema('modw'), 'systemaccount', 'sa');
+            $this->joinTo($st, "systemaccount_id", "username", "username");
+        }
+    }
+
+    /**
+     * helper function to join the data table to another table
+     */
+    private function joinTo($othertable, $joinkey, $otherkey, $colalias, $idcol = "id")
+    {
+        $this->addTable($othertable);
+        $this->addWhereCondition(new WhereCondition(new TableField($this->getDataTable(), $joinkey), '=', new TableField($othertable, $idcol)));
+        $this->addField(new TableField($othertable, $otherkey, $colalias));
+    }
+
+    public function getColumnDocumentation()
+    {
+        return $this->documentation;
+    }
+}

classes/DataWarehouse/Query/Jobs/JobMetadata.php

@@ -0,0 +1,93 @@
+<?php
+
+namespace DataWarehouse\Query\Jobs;
+
+use \XDUser;
+
+/**
+ * @see DataWarehouse::Query::iJobMetadata
+ */
+class JobMetadata extends DataWarehouse\Query\iJobMetadata
+{
+    public function getJobMetadata(XDUser $user, $jobid)
+    {
+        $job = $this->lookupJob($user, $jobid);
+        if ($job == null) {
+            return array();
+        }
+
+        return array(
+            \DataWarehouse\Query\RawQueryTypes::ACCOUNTING => true
+        );
+    }
+
+    /*
+     * Note there is no job summary data available in the Jobs realm
+     */
+    public function getJobSummary(XDUser $user, $jobid)
+    {
+        return array();
+    }
+
+    /*
+     * Note there is no job executable data available in the Jobs realm
+     */
+    public function getJobExecutableInfo(XDUser $user, $jobid)
+    {
+        return array();
+    }
+
+    /*
+     * Note there is no job timeseries data available in the Jobs realm
+     */
+    public function getJobTimeseriesMetaData(XDUser $user, $jobid)
+    {
+        return array();
+    }
+
+    /*
+     * Note there is no job timeseries data available in the Jobs realm
+     */
+    public function getJobTimeseriesMetricMeta(XDUser $user, $jobid, $tsid)
+    {
+        return array();
+    }
+
+    /*
+     * Note there is no job timeseries data available in the Jobs realm
+     */
+    public function getJobTimeseriesMetricNodeMeta(XDUser $user, $jobid, $tsid, $nodeid)
+    {
+        return array();
+    }
+
+    /*
+     * Note there is no job timeseries data available in the Jobs realm
+     */
+    public function getJobTimeseriesData(XDUser $user, $jobid, $tsid, $nodeid, $cpuid)
+    {
+        return array();
+    }
+
+    /**
+     * Lookup the job in the datawarehouse to check that it exists and the
+     * user has permission to view it.
+     *
+     * @param XDUser $user The user to lookup the job for.
+     * @param $jobid the unique identifier for the job.
+     *
+     * @return array() the accounting data for the job or null if no job exists or permission denied
+     */
+    private function lookupJob(XDUser $user, $jobid)
+    {
+        $query = new \DataWarehouse\Query\Jobs\JobDataset(array('primary_key' => $jobid));
+        $query->setMultipleRoleParameters($user->getAllRoles(), $user);
+        $stmt = $query->getRawStatement();
+
+        $job = $stmt->fetchAll(\PDO::FETCH_ASSOC);
+        if (count($job) != 1) {
+            return null;
+        }
+        return $job[0];
+    }
+}

classes/DataWarehouse/Query/Jobs/RawData.php

@@ -0,0 +1,163 @@
+<?php
+namespace DataWarehouse\Query\Jobs;
+
+use \DataWarehouse\Query\Model\Table;
+use \DataWarehouse\Query\Model\TableField;
+use \DataWarehouse\Query\Model\WhereCondition;
+use \DataWarehouse\Query\Model\Schema;
+
+/**
+ * The RawData class is reponsible for generating a query that returns
+ * the set of fact table rows given the where conditions on the aggregate
+ * table.
+ */
+class RawData extends \DataWarehouse\Query\Query
+{
+
+    public function __construct(
+        $aggregation_unit_name,
+        $start_date,
+        $end_date,
+        $group_by,
+        $stat = 'jl.jobid',
+        array $parameters = array(),
+        $query_groupname = 'query_groupname',
+        array $parameterDescriptions = array(),
+        $single_stat = false
+    ) {
+
+        parent::__construct(
+            'Jobs',
+            'modw_aggregates',
+            'jobfact',
+            array(),
+            $aggregation_unit_name,
+            $start_date,
+            $end_date,
+            null,
+            null,
+            $parameters,
+            $query_groupname,
+            $parameterDescriptions,
+            $single_stat
+        );
+
+
+        $dataTable = $this->getDataTable();
+        $joblistTable = new Table($dataTable->getSchema(), $dataTable->getName() . "_joblist", "jl");
+        $factTable = new Table(new Schema('modw'), "job_tasks", "jt" );
+
+        $resourcefactTable = new Table(new Schema('modw'), 'resourcefact', 'rf');
+        $this->addTable($resourcefactTable);
+
+        $this->addWhereCondition(new WhereCondition(
+            new TableField($dataTable, "task_resource_id"),
+            '=',
+            new TableField($resourcefactTable, "id")
+        ));
+
+        $personTable = new Table(new Schema('modw'), 'person', 'p');
+
+        $this->addTable($personTable);
+        $this->addWhereCondition(new WhereCondition(
+            new TableField($dataTable, "person_id"),
+            '=',
+            new TableField($personTable, "id")
+        ));
+
+        $this->addField(new TableField($resourcefactTable, "code", 'resource'));
+        $this->addField(new TableField($personTable, "long_name", "name"));
+
+        $this->addField(new TableField($factTable, "job_id", "jobid") );
+        $this->addField(new TableField($factTable, "local_jobid", "local_job_id") );
+
+        $this->addTable($joblistTable );
+        $this->addTable($factTable );
+
+        $this->addWhereCondition(new WhereCondition(
+            new TableField($joblistTable, "agg_id"),
+            "=",
+            new TableField($dataTable, "id")
+        ));
+        $this->addWhereCondition(new WhereCondition(
+            new TableField($joblistTable, "jobid"),
+            "=",
+            new TableField($factTable, "job_id")
+        ));
+
+        switch($stat) {
+            case "job_count":
+                $this->addWhereCondition(new WhereCondition("jt.end_time_ts", "between", "d.day_start_ts and d.day_end_ts") );
+                break;
+            case "started_job_count":
+                $this->addWhereCondition(new WhereCondition("jt.start_time_ts", "between", "d.day_start_ts and d.day_end_ts") );
+                break;
+            default:
+                // All other metrics show running job count
+                break;
+        }
+    }
+
+    /**
+     * The query differs from the base class query because the same fact table row
+     * may correspond to multiple rows in the aggregate table (e.g. a job that runs over
+     * two days). Therefore the DISTINCT keyword is added to dedupliate.
+     */
+    public function getQueryString($limit = null, $offset = null, $extraHavingClause = null)
+    {
+        $wheres = $this->getWhereConditions();
+        $groups = $this->getGroups();
+
+        $select_tables = $this->getSelectTables();
+        $select_fields = $this->getSelectFields();
+
+        $select_order_by = $this->getSelectOrderBy();
+
+        $data_query = "SELECT DISTINCT ".implode(", ", $select_fields).
+            " FROM ".implode(", ", $select_tables).
+            " WHERE ".implode(" AND ", $wheres);
+
+        if(count($groups) > 0)
+        {
+            $data_query .= " GROUP BY \n".implode(",\n", $groups);
+        }
+        if($extraHavingClause != null)
+        {
+            $data_query .= " HAVING " . $extraHavingClause . "\n";
+        }
+        if(count($select_order_by) > 0)
+        {
+            $data_query .= " ORDER BY \n".implode(",\n", $select_order_by);
+        }
+
+        if($limit !== null && $offset !== null)
+        {
+            $data_query .= " LIMIT $limit OFFSET $offset";
+        }
+        return $data_query;
+    }
+
+    /**
+     * The query differs from the base class query because the same fact table row
+     * may correspond to multiple rows in the aggregate table (e.g. a job that runs over
+     * two days). Therefore the DISTINCT keyword is added to dedupliate.
+     */
+    public function getCountQueryString()
+    {
+        $wheres = $this->getWhereConditions();
+        $groups = $this->getGroups();
+
+        $select_tables = $this->getSelectTables();
+        $select_fields = $this->getSelectFields();
+
+        $data_query = "SELECT COUNT(*) AS row_count FROM (SELECT DISTINCT ".implode(", ", $select_fields).
+            " FROM ".implode(", ", $select_tables).
+            " WHERE ".implode(" AND ", $wheres);
+
+        if(count($groups) > 0)
+        {
+            $data_query .= " GROUP BY \n".implode(",\n", $groups);
+        }
+        return $data_query . ') as a';
+    }
+}