- Added new constant BEGINNING_OF_TIME

- Added new config entry job_type_permissions_white_list in jqm.php
- Added new navigation entry jobsqueueviewer
- Added new model application/models/system/JobsQueue_model.php
- Added new option in FilterWidget for hours comparison with dates
- Added new filter core-jq-lastHour to system/filtersupdate.php
- Added new statements to system/dbupdate_3.3.php to create tables system.tbl_jobstatuses, system.tbl_jobtypes and system.tbl_jobstatuses
- Added new views application/views/system/jq/jobsQueueViewer.php and application/views/system/jq/jobsQueueViewerData.php
This commit is contained in:
Paolo
2020-03-05 15:57:06 +01:00
parent 30c6f10d80
commit cd815acdbf
16 changed files with 462 additions and 88 deletions
+7
View File
@@ -31,6 +31,13 @@ define('EXIT_VALIDATION_UDF_NOT_VALID_VAL', 17); // UDF validation has been fail
define('EXIT_AUTO_MIN', 1000); // lowest automatically-assigned error code
define('EXIT_AUTO_MAX', 2000); // highest automatically-assigned error code
/*
|--------------------------------------------------------------------------
| General purpose
|--------------------------------------------------------------------------
*/
define('BEGINNING_OF_TIME', '1970-01-01');
/*
|--------------------------------------------------------------------------
| Authentication constants
+12 -2
View File
@@ -2,5 +2,15 @@
if (! defined('BASEPATH')) exit('No direct script access allowed');
//
$config['addable_jobs_black_list'] = array('doomsday', 'sudo rm -fR /', 'sudo mv / /dev/null');
// White list of permissions that are able to store a spcific job type in database
$config['job_type_permissions_white_list'] = array(
'SAPStammdatenUpdate' => array(
'admin'
),
'OEHPayment' => array(
'admin'
),
'SAPPayment' => array(
'admin'
)
);
+7
View File
@@ -102,6 +102,13 @@ $config['navigation_header'] = array(
'expand' => true,
'sort' => 20,
'requiredPermissions' => 'system/developer:r'
),
'jobsqueueviewer' => array(
'link' => site_url('system/jq/JobsQueueViewer'),
'description' => 'Jobs Queue Viewer',
'expand' => true,
'sort' => 20,
'requiredPermissions' => 'system/developer:r'
)
)
)
@@ -1,56 +0,0 @@
<?php
if (!defined('BASEPATH')) exit('No direct script access allowed');
/**
* Jobs Queue Manager
*/
class JobsQueueMonitor extends Auth_Controller
{
/**
* Constructor
*/
public function __construct()
{
parent::__construct(
array(
'getJobsByType' => 'monitoring:r',
'getJobsByStatus' => 'monitoring:r',
'getJobsByCreationTime' => 'monitoring:r'
)
);
// Loads JobsQueueLib
$this->load->library('JobsQueueLib');
}
/**
*
*/
public function getJobsByType()
{
$jobType = $this->input->get(JobsQueueLib::PARAM_JOB_TYPE);
$this->outputJson($this->jobsqueuelib->getJobsByType($jobType));
}
/**
*
*/
public function getJobsByStatus()
{
$jobStatus = $this->input->get(JobsQueueLib::PARAM_JOB_STATUS);
$this->outputJson($this->jobsqueuelib->getJobsByStatus($jobStatus));
}
/**
*
*/
public function getJobsByCreationTime()
{
$jobCreationTime = $this->input->get(JobsQueueLib::PARAM_JOB_CREATION_TIME);
$this->outputJson($this->jobsqueuelib->getJobsByCreationTime($jobCreationTime));
}
}
@@ -35,7 +35,7 @@ class LogsViewer extends Auth_Controller
// Public methods
/**
* Main page of the InfoCenter tool
* Everything has a beginning
*/
public function index()
{
@@ -0,0 +1,51 @@
<?php
if (!defined('BASEPATH')) exit('No direct script access allowed');
/**
* This controller acts as REST JSON interface between the JobsQueueLib, that contains all the needed functionalities to
* operate with the Jobs Queue System, and other tools that cannot access directly to such library
*/
class JobsQueueManager extends Auth_Controller
{
/**
* Constructor
*/
public function __construct()
{
parent::__construct(
array(
'getJobsByType' => 'admin:r',
'addNewJobsToQueue' => 'admin:rw'
)
);
// Loads JobsQueueLib
$this->load->library('JobsQueueLib');
}
//------------------------------------------------------------------------------------------------------------------
// Public methods
/**
* To get all the most recently added jobs using the given job type
*/
public function getLastJobs()
{
$type = $this->input->get(JobsQueueLib::PARAM_JOB_TYPE);
$this->outputJson($this->jobsqueuelib->getLastJobs($type));
}
/**
* Add new jobs in the jobs queue with the given type
* jobs is an array of job objects
*/
public function addNewJobsToQueue()
{
$type = $this->input->post(JobsQueueLib::PARAM_JOB_TYPE);
$jobs = $this->input->post(JobsQueueLib::PARAM_JOBS);
$this->outputJson($this->jobsqueuelib->addNewJobsToQueue($type, $jobs));
}
}
@@ -0,0 +1,48 @@
<?php
if (!defined('BASEPATH')) exit('No direct script access allowed');
/**
* Jobs Queue Viewer
*
* This controller renders a FilterWidget to monitor the current status of the Jobs Queue System
*/
class JobsQueueViewer extends Auth_Controller
{
const PARAM_START_DATE = 'startDate';
/**
* Constructor
*/
public function __construct()
{
parent::__construct(
array(
'index' => 'system/developer:r'
)
);
// Loads WidgetLib
$this->load->library('WidgetLib');
// Loads phrases system
$this->loadPhrases(
array(
'global',
'ui',
'filter'
)
);
}
//------------------------------------------------------------------------------------------------------------------
// Public methods
/**
* Everything has a beginning
*/
public function index()
{
$this->load->view('system/jq/jobsQueueViewer.php');
}
}
+20 -13
View File
@@ -3,18 +3,22 @@
if (!defined("BASEPATH")) exit("No direct script access allowed");
/**
* Job Queue Worker
*
* This controller acts as interface of the JobsQueueLib that contains all the needed functionalities to operate with
* the Jobs Queue System
* This is an abstract class that provide basic functionalities, it has to be extended to broaden its logic
*/
abstract class JQW_Controller extends JOB_Controller
{
/**
*
* Constructor
*/
public function __construct()
{
parent::__construct();
// Loads LogLib with different ...
// Loads LogLib with different parameters
$this->load->library('LogLib', array(
'classIndex' => 5,
'functionIndex' => 5,
@@ -23,33 +27,36 @@ abstract class JQW_Controller extends JOB_Controller
'dbExecuteUser' => 'Jobs queue system'
));
// Loads JobsQueueLib
// Loads JobsQueueLib library
$this->load->library('JobsQueueLib');
}
// ------------------------------------------------------------------------------------------------------------
// Protected methods to read/write the jobs queue
//------------------------------------------------------------------------------------------------------------------
// Protected methods
/**
*
* To get all the most recently added jobs using the given job type
*/
protected function getJobsByType($jobType)
protected function getLastJobs($type)
{
$jobs = $this->jobsqueuelib->getJobsByType($jobType);
$jobs = $this->jobsqueuelib->getLastJobs($type);
if (isError($jobs)) $this->logError(getError($jobs), $jobType);
// If an error occurred then log it in database
if (isError($jobs)) $this->logError(getError($jobs), $type);
return $jobs;
}
/**
*
* Add new jobs in the jobs queue with the given type
* jobs is an array of job objects
*/
protected function addNewJobsToQueue($jobType, $jobs)
protected function addNewJobsToQueue($type, $jobs)
{
$result = $this->jobsqueuelib->addNewJobsToQueue($jobType, $jobs);
$result = $this->jobsqueuelib->addNewJobsToQueue($type, $jobs);
if (isError($result)) $this->logError(getError($result), $jobType);
// If an error occurred then log it in database
if (isError($result)) $this->logError(getError($result), $type);
return $result;
}
+5 -2
View File
@@ -93,6 +93,7 @@ class FilterWidgetLib
const OP_NOT_SET = 'nset';
// Filter options values
const OPT_HOURS = 'hours';
const OPT_DAYS = 'days';
const OPT_MONTHS = 'months';
@@ -884,7 +885,8 @@ class FilterWidgetLib
// It it's a date type
if (is_numeric($filterDefinition->condition)
&& isset($filterDefinition->option)
&& ($filterDefinition->option == self::OPT_DAYS
&& ($filterDefinition->option == self::OPT_HOURS
|| $filterDefinition->option == self::OPT_DAYS
|| $filterDefinition->option == self::OPT_MONTHS))
{
$condition = '< (NOW() - \''.$filterDefinition->condition.' '.$filterDefinition->option.'\'::interval)';
@@ -899,7 +901,8 @@ class FilterWidgetLib
// It it's a date type
if (is_numeric($filterDefinition->condition)
&& isset($filterDefinition->option)
&& ($filterDefinition->option == self::OPT_DAYS
&& ($filterDefinition->option == self::OPT_HOURS
|| $filterDefinition->option == self::OPT_DAYS
|| $filterDefinition->option == self::OPT_MONTHS))
{
$condition = '> (NOW() - \''.$filterDefinition->condition.' '.$filterDefinition->option.'\'::interval)';
+18 -12
View File
@@ -2,27 +2,33 @@
if (!defined('BASEPATH')) exit('No direct script access allowed');
/**
* Library that contains all the needed functionalities to operate with the Jobs Queue System
*/
class JobsQueueLib
{
//
const STATUS_RUNNING = 'running';
const STATUS_NEW = 'new';
const STATUS_DONE = 'done';
//
const PARAM_JOB_TYPE = 'jobType';
const PARAM_JOB_STATUS = 'jobStatus';
const PARAM_JOB_CREATION_TIME = 'jobCreatinTime';
//
// Job types
// SAP
const JOB_TYPE_SAP_STAMMDATEN_UPDATE = 'SAPStammdatenUpdate';
const JOB_TYPE_SAP_PAYMENT = 'SAPPayment';
// DVUH
const JOB_TYPE_OEH_PAYMENT = 'OEHPayment';
// Job statuses
const STATUS_NEW = 'new';
const STATUS_RUNNING = 'running';
const STATUS_DONE = 'done';
const STATUS_FAILED = 'failed';
// Parameter names
const PARAM_JOB_TYPE = 'type';
const PARAM_JOB_STATUS = 'status';
const PARAM_JOBS = 'jobs';
private $_ci; // CI instance
/**
* Construct
* Constructor
*/
public function __construct($authenticate = true)
{
@@ -0,0 +1,15 @@
<?php
class JobsQueue_model extends DB_Model
{
/**
* Constructor
*/
public function __construct()
{
parent::__construct();
$this->dbTable = 'system.tbl_jobsqueue';
$this->pk = 'jobid';
}
}
@@ -0,0 +1,47 @@
<?php
$this->load->view(
'templates/FHC-Header',
array(
'title' => 'Jobs Queue Viewer',
'jquery' => true,
'jqueryui' => true,
'bootstrap' => true,
'fontawesome' => true,
'sbadmintemplate' => true,
'tablesorter' => true,
'ajaxlib' => true,
'filterwidget' => true,
'navigationwidget' => true,
'phrases' => array(
'global' => array('mailAnXversandt'),
'ui' => array('bitteEintragWaehlen')
),
'customCSSs' => 'public/css/sbadmin2/tablesort_bootstrap.css',
'customJSs' => array('public/js/bootstrapper.js')
)
);
?>
<body>
<div id="wrapper">
<?php echo $this->widgetlib->widget('NavigationWidget'); ?>
<div id="page-wrapper">
<div class="container-fluid">
<div class="row">
<div class="col-lg-12">
<h3 class="page-header">
Jobs Queue Viewer
</h3>
</div>
</div>
<div>
<?php $this->load->view('system/jq/jobsQueueViewerData.php'); ?>
</div>
</div>
</div>
</div>
</body>
<?php $this->load->view('templates/FHC-Footer'); ?>
@@ -0,0 +1,67 @@
<?php
$filterWidgetArray = array(
'query' => '
SELECT jq.jobid AS "JobId",
jq.creationtime AS "CreationTime",
jq.type AS "Type",
jq.status AS "Status",
jq.starttime AS "StartTime",
jq.endtime AS "EndTime",
jq.insertvon AS "UserService"
FROM system.tbl_jobsqueue jq
ORDER BY jq.creationtime DESC, jq.starttime DESC, jq.endtime DESC
',
'requiredPermissions' => 'admin',
'datasetRepresentation' => 'tablesorter',
'columnsAliases' => array(
'Job id',
'Creation time',
'Type',
'Status',
'Start time',
'End time',
'User/Service'
),
'formatRow' => function($datasetRaw) {
$datasetRaw->CreationTime = date_format(date_create($datasetRaw->CreationTime), 'd.m.Y H:i:s');
$datasetRaw->StartTime = date_format(date_create($datasetRaw->StartTime), 'd.m.Y H:i:s');
$datasetRaw->EndTime = date_format(date_create($datasetRaw->EndTime), 'd.m.Y H:i:s');
return $datasetRaw;
},
'markRow' => function($datasetRaw) {
$mark = '';
if ($datasetRaw->Status == 'Failed')
{
$mark = 'text-red';
}
if ($datasetRaw->Status == 'Done')
{
$mark = 'text-green';
}
if ($datasetRaw->Status == 'Running')
{
$mark = 'text-orange';
}
if ($datasetRaw->Status == 'New')
{
$mark = 'text-info';
}
return $mark;
}
);
$filterWidgetArray['app'] = 'core';
$filterWidgetArray['datasetName'] = 'jq';
$filterWidgetArray['filter_id'] = $this->input->get('filter_id');
echo $this->widgetlib->widget('FilterWidget', $filterWidgetArray);
?>
+2 -2
View File
@@ -2,7 +2,7 @@
$this->load->view(
'templates/FHC-Header',
array(
'title' => 'Logs viewer',
'title' => 'Logs Viewer',
'jquery' => true,
'jqueryui' => true,
'bootstrap' => true,
@@ -32,7 +32,7 @@
<div class="row">
<div class="col-lg-12">
<h3 class="page-header">
JobsViewer
Job Logs Viewer
</h3>
</div>
</div>
+131
View File
@@ -3842,6 +3842,134 @@ if ($result = $db->db_query("SELECT 1 FROM pg_class WHERE relname = 'unq_idx_abl
}
}
// Creates table system.tbl_jobstatuses if it doesn't exist and grants privileges
if (!$result = @$db->db_query('SELECT 1 FROM system.tbl_jobstatuses LIMIT 1'))
{
$qry = 'CREATE TABLE system.tbl_jobstatuses (
status character varying NOT NULL
);
COMMENT ON TABLE system.tbl_jobstatuses IS \'All possible job statuses\';
COMMENT ON COLUMN system.tbl_jobstatuses.status IS \'Job status value and primary key\';
ALTER TABLE ONLY system.tbl_jobstatuses ADD CONSTRAINT pk_jobstatuses PRIMARY KEY (status);
';
if (!$db->db_query($qry))
echo '<strong>system.tbl_jobstatuses: '.$db->db_last_error().'</strong><br>';
else
echo '<br>system.tbl_jobstatuses table created';
$qry = 'GRANT SELECT ON TABLE system.tbl_jobstatuses TO web;';
if (!$db->db_query($qry))
echo '<strong>system.tbl_jobstatuses: '.$db->db_last_error().'</strong><br>';
else
echo '<br>Granted privileges to <strong>web</strong> on system.tbl_jobstatuses';
$qry = 'GRANT SELECT, UPDATE, INSERT, DELETE ON TABLE system.tbl_jobstatuses TO vilesci;';
if (!$db->db_query($qry))
echo '<strong>system.tbl_jobstatuses: '.$db->db_last_error().'</strong><br>';
else
echo '<br>Granted privileges to <strong>vilesci</strong> on system.tbl_jobstatuses';
}
// Creates table system.tbl_jobtypes if it doesn't exist and grants privileges
if (!$result = @$db->db_query('SELECT 1 FROM system.tbl_jobtypes LIMIT 1'))
{
$qry = 'CREATE TABLE system.tbl_jobtypes (
type character varying(256) NOT NULL,
description text NOT NULL
);
COMMENT ON TABLE system.tbl_jobtypes IS \'All possible job types\';
COMMENT ON COLUMN system.tbl_jobtypes.type IS \'Job type value and primary key\';
COMMENT ON COLUMN system.tbl_jobtypes.description IS \'Job type description\';
ALTER TABLE ONLY system.tbl_jobtypes ADD CONSTRAINT pk_jobtypes PRIMARY KEY (type);
';
if (!$db->db_query($qry))
echo '<strong>system.tbl_jobtypes: '.$db->db_last_error().'</strong><br>';
else
echo '<br>system.tbl_jobtypes table created';
$qry = 'GRANT SELECT ON TABLE system.tbl_jobtypes TO web;';
if (!$db->db_query($qry))
echo '<strong>system.tbl_jobtypes: '.$db->db_last_error().'</strong><br>';
else
echo '<br>Granted privileges to <strong>web</strong> on system.tbl_jobtypes';
$qry = 'GRANT SELECT, UPDATE, INSERT, DELETE ON TABLE system.tbl_jobtypes TO vilesci;';
if (!$db->db_query($qry))
echo '<strong>system.tbl_jobtypes: '.$db->db_last_error().'</strong><br>';
else
echo '<br>Granted privileges to <strong>vilesci</strong> on system.tbl_jobtypes';
}
// Creates table system.tbl_jobsqueue if it doesn't exist and grants privileges
if (!$result = @$db->db_query('SELECT 1 FROM system.tbl_jobsqueue LIMIT 1'))
{
$qry = 'CREATE TABLE system.tbl_jobsqueue (
jobid integer NOT NULL,
type character varying(256) NOT NULL,
creationtime timestamp without time zone DEFAULT now(),
status character varying(64) NOT NULL,
input jsonb,
output jsonb,
starttime timestamp without time zone,
endtime timestamp without time zone,
insertvon character varying(32),
insertamum timestamp without time zone DEFAULT now()
);
COMMENT ON TABLE system.tbl_jobsqueue IS \'Table to schedule/manage the jobs queue\';
COMMENT ON COLUMN system.tbl_jobsqueue.jobid IS \'Primary key\';
COMMENT ON COLUMN system.tbl_jobsqueue.type IS \'Job type\';
COMMENT ON COLUMN system.tbl_jobsqueue.creationtime IS \'Job creation timestamp\';
COMMENT ON COLUMN system.tbl_jobsqueue.status IS \'Job current status\';
COMMENT ON COLUMN system.tbl_jobsqueue.input IS \'Job input in JSON format\';
COMMENT ON COLUMN system.tbl_jobsqueue.output IS \'Job output in JSON format\';
COMMENT ON COLUMN system.tbl_jobsqueue.starttime IS \'Job start timestamp\';
COMMENT ON COLUMN system.tbl_jobsqueue.endtime IS \'Job end timestamp\';
COMMENT ON COLUMN system.tbl_jobsqueue.insertvon IS \'User/Service who/that inserted this record\';
COMMENT ON COLUMN system.tbl_jobsqueue.insertamum IS \'Record insert time stamp\';
CREATE SEQUENCE system.seq_jobsqueue_jobid
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE system.seq_jobsqueue_jobid OWNED BY system.tbl_jobsqueue.jobid;
ALTER TABLE ONLY system.tbl_jobsqueue ALTER COLUMN jobid SET DEFAULT nextval(\'system.seq_jobsqueue_jobid\'::regclass);
ALTER TABLE ONLY system.tbl_jobsqueue ADD CONSTRAINT pk_jobsqueue PRIMARY KEY (jobid);
ALTER TABLE ONLY system.tbl_jobsqueue ADD CONSTRAINT fk_jobsqueue_status FOREIGN KEY (status) REFERENCES system.tbl_jobstatuses(status) ON UPDATE CASCADE ON DELETE RESTRICT;
ALTER TABLE ONLY system.tbl_jobsqueue ADD CONSTRAINT fk_jobsqueue_type FOREIGN KEY (type) REFERENCES system.tbl_jobtypes(type) ON UPDATE CASCADE ON DELETE RESTRICT;
';
if (!$db->db_query($qry))
echo '<strong>system.tbl_jobsqueue: '.$db->db_last_error().'</strong><br>';
else
echo '<br>system.tbl_jobsqueue table created';
$qry = 'GRANT SELECT ON TABLE system.tbl_jobsqueue TO web;';
if (!$db->db_query($qry))
echo '<strong>system.tbl_jobsqueue: '.$db->db_last_error().'</strong><br>';
else
echo '<br>Granted privileges to <strong>web</strong> on system.tbl_jobsqueue';
$qry = 'GRANT SELECT, UPDATE, INSERT, DELETE ON TABLE system.tbl_jobsqueue TO vilesci;';
if (!$db->db_query($qry))
echo '<strong>system.tbl_jobsqueue: '.$db->db_last_error().'</strong><br>';
else
echo '<br>Granted privileges to <strong>vilesci</strong> on system.tbl_jobsqueue';
}
// *** Pruefung und hinzufuegen der neuen Attribute und Tabellen
echo '<H2>Pruefe Tabellen und Attribute!</H2>';
@@ -4100,6 +4228,9 @@ $tabellen=array(
"system.tbl_log" => array("log_id","person_id","zeitpunkt","app","oe_kurzbz","logtype_kurzbz","logdata","insertvon","taetigkeit_kurzbz"),
"system.tbl_logtype" => array("logtype_kurzbz", "data_schema"),
"system.tbl_filters" => array("filter_id","app","dataset_name","filter_kurzbz","person_id","description","sort","default_filter","filter","oe_kurzbz","statistik_kurzbz"),
"system.tbl_jobsqueue" => array("jobid", "type", "creationtime", "status", "input", "output", "starttime", "endtime", "insertvon", "insertamum"),
"system.tbl_jobstatuses" => array("status"),
"system.tbl_jobtypes" => array("type", "description"),
"system.tbl_phrase" => array("phrase_id","app","phrase","insertamum","insertvon","category"),
"system.tbl_phrasentext" => array("phrasentext_id","phrase_id","sprache","orgeinheit_kurzbz","orgform_kurzbz","text","description","insertamum","insertvon"),
"system.tbl_rolle" => array("rolle_kurzbz","beschreibung"),
+31
View File
@@ -631,6 +631,37 @@ $filters = array(
}
',
'oe_kurzbz' => null,
),
array(
'app' => 'core',
'dataset_name' => 'jq',
'filter_kurzbz' => 'lastHour',
'description' => '{Last hour queued jobs}',
'sort' => 1,
'default_filter' => true,
'filter' => '
{
"name": "All jobs queued in the last hour",
"columns": [
{"name": "JobId"},
{"name": "CreationTime"},
{"name": "Type"},
{"name": "Status"},
{"name": "StartTime"},
{"name": "EndTime"},
{"name": "UserService"}
],
"filters": [
{
"name": "CreationTime",
"operation": "lt",
"condition": "1",
"option": "hours"
}
]
}
',
'oe_kurzbz' => null,
)
);