Skip to content
Snippets Groups Projects
Commit 0df58f6d authored by Andreas Åkre Solberg's avatar Andreas Åkre Solberg
Browse files

Fixing the statistics module. Adding classes and tidy up. config option for log syntax. 

git-svn-id: https://simplesamlphp.googlecode.com/svn/trunk@1002 44740490-163a-0410-bde0-09ae8108e29a
parent 8fc74b3b
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
modules/statistics/config-templates/statistics.php
+ 25
− 6
View file @ 0df58f6d
......@@ -6,9 +6,12 @@
$config = array (
'statdir' => '/tmp/stats/',
'inputfile' => '/Users/andreas/Desktop/stat2.log',
# 'inputfile' => '/Users/andreas/Desktop/simplesamlphp.stat.1',
'offset' => 60*60*1, // Two hours offset to match epoch and norwegian winter time.
'inputfile' => '/var/log/simplesamlphp.stat',
'offset' => 60*60*2, // Two hours offset to match epoch and norwegian winter time.
'datestart' => 1,
'datelength' => 15,
'offsetspan' => 21,
/*
* Do you want to generate statistics using the cron module? If so, specify which cron tag to use.
......@@ -31,14 +34,28 @@ $config = array (
'dateformat-period' => 'j. M', // 4. Mars
'dateformat-intra' => 'j. M H:i', // 4. Mars 12:30
# 'dateformat-intra' => 'j. H:i', // 4. Mars 12:30
),
'sso_day80' => array(
'name' => 'Number of SP logins (per day spanning 80 days)',
'descr' => 'The number of Service Provider logins put into slots of one hour.',
'action' => 'saml20-sp-SSO',
'col' => 5, // Service Provider EntityID
'slot' => 60*60*24, // Slots of six hour
'fileslot' => 60*60*24*80, // 7 days of data in each file
'axislabelint' => 7, // Number of slots per label. 24 is one each day
'dateformat-period' => 'j. M', // 4. Mars
'dateformat-intra' => 'j. M', # H:i', // 4. Mars 12:30
# 'dateformat-intra' => 'j. H:i', // 4. Mars 12:30
),
'sso_hoursweek' => array(
'name' => 'Numer of SP logins (per hour)',
'name' => 'Numer of SP logins (per hour spanning 7 days)',
'descr' => 'The number of Service Provider logins put into slots of one hour.',
'action' => 'saml20-sp-SSO',
'col' => 7, // Service Provider EntityID
'col' => 5, // Service Provider EntityID
'slot' => 60*60, // Slots of one hour
'fileslot' => 60*60*24*7, // 7 days of data in each file
'axislabelint' => 24, // Number of slots per label. 24 is one each day
......@@ -52,7 +69,7 @@ $config = array (
'descr' => 'The number of Service Provider logins put into slots of one day.',
'action' => 'saml20-sp-SSO',
'col' => 7, // Service Provider EntityID
'col' => 5, // Service Provider EntityID
'slot' => 60*60*24, // Slots of one day
'fileslot' => 60*60*24*30, // 30 days of data in each file
'axislabelint' => 7, // Number of slots per label. 24 is one each day
......@@ -61,6 +78,8 @@ $config = array (
'dateformat-intra' => 'j. M', // 4. Mars 12:30
# 'dateformat-intra' => 'j. H:i', // 4. Mars 12:30
),
),
);
......
modules/statistics/extlibs/loganalyzer.php
+ 32
− 58
View file @ 0df58f6d
<?php
#int mktime ([ int $hour [, int $minute [, int $second [, int $month [, int $day [, int $year [, int $is_dst ]]]]]]] )
#http://chart.apis.google.com/chart?cht=lc&chs=200x125&chd=s:helloWorld&chxt=x,y&chxl=0:|Mar|Apr|May|June|July|1:||50+Kb
$config = SimpleSAML_Configuration::getInstance();
$statconfig = $config->copyFromBase('statconfig', 'statistics.php');
$statdir = $statconfig->getValue('statdir');
$offset = $statconfig->getValue('offset');
$inputfile = $statconfig->getValue('inputfile');
$statrules = $statconfig->getValue('statrules');
$file = fopen($inputfile, 'r');
$logfile = file($inputfile, FILE_IGNORE_NEW_LINES );
# Aug 27 12:54:25 ssp 5 STAT [5416262207] saml20-sp-SSO urn:mace:feide.no:services:no.uninett.wiki-feide sam.feide.no NA
#
#Oct 30 11:07:14 www1 simplesamlphp-foodle[12677]: 5 STAT [200b4679af] saml20-sp-SLO spinit urn:mace:feide.no:services:no.feide.foodle sam.feide.no
function parse15($str) {
$di = date_parse($str);
$datestamp = mktime($di['hour'], $di['minute'], $di['second'], $di['month'], $di['day']);
return $datestamp;
}
function parse23($str) {
$timestamp = strtotime($str);
return $timestamp;
}
$logparser = new sspmod_statistics_LogParser(
$statconfig->getValue('datestart', 0), $statconfig->getValue('datelength', 15), $statconfig->getValue('offsetspan', 44)
);
$datehandler = new sspmod_statistics_DateHandler($statconfig->getValue('offset', 0));
$results = array();
# Sat, 16 Feb 08 00:55:11 (23 chars)
// Parse through log file, line by line
foreach ($logfile AS $logline) {
$datenumbers = 19;
if (!preg_match('/^[0-9]{4}/', $logline)) continue;
$datestr = substr($logline,0,$datenumbers);
#$datestr = substr($logline,0,23);
$timestamp = parse15($datestr) + $offset;
$restofline = substr($logline,$datenumbers+1);
$restcols = split(' ', $restofline);
$action = $restcols[5];
// print_r($timestamp);
// print_r($restcols); if ($i++ > 5) exit;
// Continue if STAT is not found on line.
if (!preg_match('/STAT/', $logline)) continue;
// Parse log, and extract epoch time and rest of content.
$epoch = $logparser->parseEpoch($logline);
$content = $logparser->parseContent($logline);
$action = $content[4];
// Iterate all the statrules from config.
foreach ($statrules AS $rulename => $rule) {
$timeslot = floor($timestamp/$rule['slot']);
$fileslot = floor($timestamp/$rule['fileslot']);
if (isset($rule['action'])) {
if ($action !== $rule['action']) continue;
}
$difcol = $restcols[$rule['col']];
$difcolsp = split('@', $difcol);
$difcol = $difcolsp[1];
// print(' foo: ' . $difcol . ' : ' . $rule['col']); exit;
$timeslot = $datehandler->toSlot($epoch, $rule['slot']);
$fileslot = $datehandler->toSlot($epoch, $rule['fileslot']); //print_r($content);
if (isset($rule['action']) && ($action !== $rule['action'])) continue;
$difcol = $content[$rule['col']]; // echo '[...' . $difcol . '...]';
$results[$rulename][$fileslot][$timeslot]['_']++;
$results[$rulename][$fileslot][$timeslot][$difcol]++;
}
}
// Iterate the first level of results, which is per rule, as defined in the config.
foreach ($results AS $rulename => $ruleresults) {
// Iterate the second level of results, which is the fileslot.
foreach ($ruleresults AS $fileno => $fileres) {
$slotlist = array_keys($fileres);
$start = $slotlist[0];
$start = $fileno*($statrules[$rulename]['fileslot'] / $statrules[$rulename]['slot']);
#echo 'Start was set to ' . $start . ' instead consider ' . $fileno*($statrules[$rulename]['fileslot'] / $statrules[$rulename]['slot']) . "\n";
$end = $slotlist[count($fileres)-1];
$end = ($fileno+1)*($statrules[$rulename]['fileslot'] / $statrules[$rulename]['slot']);
#echo 'End was set to ' . $end . ' instead consider ' . ($fileno+1)*($statrules[$rulename]['fileslot'] / $statrules[$rulename]['slot']) . "\n";
// exit;
// echo "From $start to $end \n";
// Get start and end slot number within the file, based on the fileslot.
$start = $datehandler->toSlot($datehandler->fromSlot($fileno, $statrules[$rulename]['fileslot']), $statrules[$rulename]['slot']);
$end = $datehandler->toSlot($datehandler->fromSlot($fileno+1, $statrules[$rulename]['fileslot']), $statrules[$rulename]['slot']);
// Fill in missing entries and sort file results
$filledresult = array();
for ($slot = $start; $slot < $end; $slot++) {
$filledresult[$slot] = (isset($fileres[$slot])) ? $fileres[$slot] : array('_' => 0);
}
// store file
file_put_contents($statdir . $rulename . '-' . $fileno . '.stat', serialize($filledresult) );
}
}
......
modules/statistics/lib/DateHandler.php 0 → 100644
+ 49
− 0
View file @ 0df58f6d
<?php
/*
* @author Andreas Åkre Solberg <andreas.solberg@uninett.no>
* @package simpleSAMLphp
* @version $Id$
*/
class sspmod_statistics_DateHandler {
private $offset;
/**
* Constructor
*
* @param array $offset Date offset
*/
public function __construct($offset) {
$this->offset = $offset;
}
public function toSlot($epoch, $slotsize) {
return floor( ($epoch + $this->offset) / $slotsize);
}
public function fromSlot($slot, $slotsize) {
return $slot*$slotsize - $this->offset;
}
public function prettyDateEpoch($epoch, $dateformat) {
return date($dateformat, $epoch);
}
public function prettyDateSlot($slot, $slotsize, $dateformat) {
return $this->prettyDateEpoch($this->fromSlot($slot, $slotsize), $dateformat);
}
}
// $datestr = substr($logline,0,$datenumbers);
// #$datestr = substr($logline,0,23);
// $timestamp = parse15($datestr) + $offset;
// $restofline = substr($logline,$datenumbers+1);
// $restcols = split(' ', $restofline);
// $action = $restcols[5];
// print_r($timestamp);
// print_r($restcols); if ($i++ > 5) exit;
?>
\ No newline at end of file
modules/statistics/lib/Graph/GoogleCharts.php 0 → 100644
+ 58
− 0
View file @ 0df58f6d
<?php
/*
* @author Andreas Åkre Solberg <andreas.solberg@uninett.no>
* @package simpleSAMLphp
* @version $Id$
*/
class sspmod_statistics_Graph_GoogleCharts {
private $x, $y;
/**
* Constructor
*/
public function __construct($x, $y) {
$this->x = $x; $this->y = $y;
}
private function encodeaxis($axis) {
return join('|', $axis);
}
# t:10.0,58.0,95.0
private function encodedata($data) {
return 't:' . join(',', $data);
}
public function show($axis, $axispos, $values, $max) {
$nv = count($values);
$url = 'http://chart.apis.google.com/chart?' .
'chs=800x350' .
'&chd=' . $this->encodedata($values) .
'&cht=lc' .
'&chxt=x,y' .
'&chxl=0:|' . $this->encodeaxis($axis) . # . $'|1:||top' .
'&chxp=0,' . join(',', $axispos) .
# '&chxp=0,0.3,0.4' .
'&chxr=0,0,1|1,0,' . $max .
# '&chm=R,CCCCCC,0,0.25,0.5' .
'&chg=' . (2400/(count($values)-1)) . ',20,3,3'; // lines
return $url;
}
public static function roof($in) {
if ($in < 1) return 1;
$base = log10($in);
$r = ceil(5*$in / pow(10, ceil($base)));
return ($r/5)*pow(10, ceil($base));
}
// $foo = array(0, 2, 2.3, 2.6, 6, 10, 15, 98, 198, 256, 487, 563, 763, 801, 899, 999, 987, 198234.485, 283746);
// foreach ($foo AS $f) {
// echo '<p>' . $f . ' => ' . roof($f);
// }
// exit;
}
?>
\ No newline at end of file
modules/statistics/lib/LogParser.php 0 → 100644
+ 58
− 0
View file @ 0df58f6d
<?php
/*
* @author Andreas Åkre Solberg <andreas.solberg@uninett.no>
* @package simpleSAMLphp
* @version $Id$
*/
class sspmod_statistics_LogParser {
private $datestart;
private $datelength;
private $offset;
/**
* Constructor
*
* @param $datestart At which char is the date starting
* @param $datelength How many characters is the date (on the b
* @param $offset At which char is the rest of the entries starting
*/
public function __construct($datestart, $datelength, $offset) {
$this->datestart = $datestart;
$this->datelength = $datelength;
$this->offset = $offset;
}
public function parseEpoch($line) {
$epoch = strtotime(substr($line, 0, $this->datelength));
echo 'debug ' . $line . "\n";
echo 'debug [' . substr($line, 0, $this->datelength) . '] => [' . $epoch . ']' . "\n";
return $epoch;
}
public function parseContent($line) {
$contentstr = substr($line, $this->offset);
$content = split(' ', $contentstr);
return $content;
}
# Aug 27 12:54:25 ssp 5 STAT [5416262207] saml20-sp-SSO urn:mace:feide.no:services:no.uninett.wiki-feide sam.feide.no NA
#
#Oct 30 11:07:14 www1 simplesamlphp-foodle[12677]: 5 STAT [200b4679af] saml20-sp-SLO spinit urn:mace:feide.no:services:no.feide.foodle sam.feide.no
function parse15($str) {
$di = date_parse($str);
$datestamp = mktime($di['hour'], $di['minute'], $di['second'], $di['month'], $di['day']);
return $datestamp;
}
function parse23($str) {
$timestamp = strtotime($str);
return $timestamp;
}
}
?>
\ No newline at end of file
modules/statistics/www/showstats.php
+ 42
− 111
View file @ 0df58f6d
<?php
function encodeaxis($axis) {
return join('|', $axis);
}
# t:10.0,58.0,95.0
function encodedata($data) {
return 't:' . join(',', $data);
}
function show($axis, $axispos, $values, $max) {
$nv = count($values);
$url = 'http://chart.apis.google.com/chart?' .
'chs=800x350' .
'&chd=' . encodedata($values) .
'&cht=lc' .
'&chxt=x,y' .
'&chxl=0:|' . encodeaxis($axis) . # . $'|1:||top' .
'&chxp=0,' . join(',', $axispos) .
# '&chxp=0,0.3,0.4' .
'&chxr=0,0,1|1,0,' . $max .
# '&chm=R,CCCCCC,0,0.25,0.5' .
'&chg=' . (2400/(count($values)-1)) . ',20,3,3'; // lines
return $url;
}
function prettydate($timeslot, $granularity, $offset, $dateformat) {
# echo 'date: [' . $dateformat . date($dateformat, $timeslot*$granularity-$offset) . ']';
return date($dateformat, $timeslot*$granularity-$offset);
}
function roof($in) {
if ($in < 1) return 1;
$base = log10($in);
$r = ceil(5*$in / pow(10, ceil($base)));
return ($r/5)*pow(10, ceil($base));
}
// $foo = array(0, 2, 2.3, 2.6, 6, 10, 15, 98, 198, 256, 487, 563, 763, 801, 899, 999, 987, 198234.485, 283746);
// foreach ($foo AS $f) {
// echo '<p>' . $f . ' => ' . roof($f);
// }
// exit;
$config = SimpleSAML_Configuration::getInstance();
$statconfig = $config->copyFromBase('statconfig', 'statistics.php');
$statdir = $statconfig->getValue('statdir');
$offset = $statconfig->getValue('offset');
$inputfile = $statconfig->getValue('inputfile');
$statrules = $statconfig->getValue('statrules');
$datehandler = new sspmod_statistics_DateHandler($statconfig->getValue('offset', 0));
/*
* Walk through file lists, and get available [rule][fileslot]...
*/
if (!is_dir($statdir))
throw new Exception('Statisics output directory [' . $statdir . '] does not exists.');
$filelist = scandir($statdir);
$available = array();
foreach ($filelist AS $file) {
if (preg_match('/([a-z_]+)-([0-9]+)\.stat/', $file, $matches)) {
if (preg_match('/([a-z0-9_]+)-([0-9]+)\.stat/', $file, $matches)) {
if (array_key_exists($matches[1], $statrules)) {
$available[$matches[1]][] = $matches[2];
}
}
}
/*
* Create array with information about available rules..
*/
$available_rules = array();
foreach ($available AS $key => $av) {
$available_rules[$key] = array('name' => $statrules[$key]['name'], 'descr' => $statrules[$key]['descr']);
}
$availrulenames = array_keys($available_rules);
// Get selected rulename....
$rule = $availrulenames[0];
if(array_key_exists('rule', $_GET)) {
if (array_key_exists($_GET['rule'], $available_rules)) {
......@@ -86,34 +43,29 @@ if(array_key_exists('rule', $_GET)) {
}
}
/*
* Get list of avaiable times in current file (rule)
*/
$available_times = array();
foreach ($available[$rule] AS $slot) {
$available_times[$slot] = prettydate($slot, $statrules[$rule]['fileslot'], $offset, $statrules[$rule]['dateformat-period']) . ' to ' .
prettydate($slot+1, $statrules[$rule]['fileslot'], $offset, $statrules[$rule]['dateformat-period']);
$available_times[$slot] = $datehandler->prettyDateSlot($slot, $statrules[$rule]['fileslot'], $statrules[$rule]['dateformat-period']) .
' to ' . $datehandler->prettyDateSlot($slot+1, $statrules[$rule]['fileslot'], $statrules[$rule]['dateformat-period']);
}
#print_r($available_times); exit;
// Get which time (fileslot) to use.. First get a default, which is the most recent one.
$fileslot = $available[$rule][count($available[$rule])-1];
// Then check if the user have provided one.
if (array_key_exists('time', $_GET)) {
if (in_array($_GET['time'], $available[$rule])) {
$fileslot = $_GET['time'];
}
}
#echo 'fileslot: ' . $fileslot; exit;
#echo '<pre>'; print_r($available_rules); exit;
#echo '<pre>'; print_r($available); exit;
// Get file and extract results.
$resultfile = file_get_contents($statdir . $rule . '-' . $fileslot . '.stat');
$results = unserialize($resultfile);
// echo '<html><body><pre>';
// print_r($results);
// echo '</pre>';
$dataset = array();
$axis = array();
......@@ -134,58 +86,60 @@ if (isset($_REQUEST['d'])) {
$delimiter = $_REQUEST['d'];
}
/*
* Walk through dataset to get the max values.
*/
$maxvalue = 0;
$maxvaluetime = 0;
$debugdata = array();
foreach($results AS $slot => $res) {
if ($res[$delimiter] > $maxvalue) { $maxvaluetime = prettydate($slot, $statrules[$rule]['slot'], $offset, $statrules[$rule]['dateformat-intra']); }
if ($res[$delimiter] > $maxvalue) {
$maxvaluetime = $datehandler->prettyDateSlot($slot, $slotsize, $dateformat_intra);
}
$maxvalue = max($res[$delimiter],$maxvalue);
$debugdata[] = array(
prettydate($slot, $statrules[$rule]['slot'], $offset, $statrules[$rule]['dateformat-intra']),
$res[$delimiter]
);
$debugdata[] = array($datehandler->prettyDateSlot($slot, $slotsize, $dateformat_intra), $res[$delimiter] );
}
$max = roof($maxvalue);
$max = sspmod_statistics_Graph_GoogleCharts::roof($maxvalue);
#echo 'Maxvalue [' . $maxvalue . '] at time ' . $maxvaluetime; exit;
#echo '<pre>'; print_r($debugdata); exit;
#echo '<pre>'; print_r($debugdata); exit;
/*
* Walk through dataset to get percent values from max into dataset[].
*/
$availdelimiters = array();
$lastslot = 0;
$xentries = count($results);
$i = 0;
foreach($results AS $slot => $res) {
$lastslot = 0; $i = 0;
#echo '<p>' . date($dateformat, $slot*$granularity) . ': ' . (isset($results[$slot]) ? $results[$slot] : 0);
foreach($results AS $slot => $res) {
$dataset[] = number_format(100*$res[$delimiter] / $max, 2);
foreach(array_keys($res) AS $nd) $availdelimiters[$nd] = 1;
#$dataset[] = (isset($results[$slot]) ? round(($results[$slot]*$perseconds/($granularity*$max))*100) : 0);
if ($slot % $axislabelint == 0) {
$axis[] = date($dateformat_intra, $slot*$slotsize - $offset);
// check if there should be an axis here...
if ( $slot % $axislabelint == 0) {
$axis[] = $datehandler->prettyDateSlot($slot, $slotsize, $dateformat_intra);
$axispos[] = (($i)/($xentries-1));
#echo "<p> ". $slot . " = " . date($dateformat_intra, ($slot*$slotsize - $offset) ) . " ";
#echo 'set axis on [' . $slot . ']';
}
#echo "<p> ". $slot . " = " . date($dateformat_intra, ($slot*$slotsize - $offset) ) . " ";
$lastslot = $slot;
$i++;
}
#echo 'set axis on lastslot [' . $lastslot . ']';
$axis[] = date($dateformat_intra, ($lastslot*$slotsize) + $slotsize - $offset);
#echo "<p> ". ($lastslot+1) . " = " . date($dateformat_intra, (($lastslot+1)*$slotsize - $offset) ) . " ";
#print_r($axis);
// echo '<input value="' . htmlspecialchars(show($axis, $dataset, $max)) . '" />';
// echo '<img src="' . htmlspecialchars() . '" />';
$grapher = new sspmod_statistics_Graph_GoogleCharts(800, 350);
$t = new SimpleSAML_XHTML_Template($config, 'statistics:statistics-tpl.php');
$t->data['header'] = 'stat';
$t->data['imgurl'] = show($axis, $axispos, $dataset, $max);
$t->data['imgurl'] = $grapher->show($axis, $axispos, $dataset, $max);
$t->data['available.rules'] = $available_rules;
$t->data['available.times'] = $available_times;
$t->data['selected.rule']= $rule;
......@@ -195,28 +149,5 @@ $t->data['availdelimiters'] = array_keys($availdelimiters);
$t->show();
// $slotlist = array_keys($results);
// $start = $slotlist[0];
// $end = $slotlist[count($results)-1];
//
// #echo 'from slot ' . $start . ' to ' . $end;
//
// $dataset = array();
// $axis = array();
// $max = 10;
//
// $perseconds = 60;
//
// for ($slot = $start; $slot <= $end; $slot++) {
// #echo '<p>' . date($dateformat, $slot*$granularity) . ': ' . (isset($results[$slot]) ? $results[$slot] : 0);
//
// $dataset[] = (isset($results[$slot]) ? round(($results[$slot]*$perseconds/($granularity*$max))*100) : 0);
// if ($slot % 3 == 0)
// $axis[] = date($dateformat, $slot*$granularity);
// }
//
// echo '<input value="' . htmlspecialchars(show($axis, $dataset)) . '" />';
// echo '<img src="' . htmlspecialchars(show($axis, $dataset, $max)) . '" />';
?>
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment