diff --git a/modules/statistics/bin/loganalyzer.php b/modules/statistics/bin/loganalyzer.php index 644b5d40dcb5c97fb7a3b5871b597fcd5d15e6b8..79e242c38a1e92675f5c73d75b47a095e93df8c6 100755 --- a/modules/statistics/bin/loganalyzer.php +++ b/modules/statistics/bin/loganalyzer.php @@ -53,7 +53,9 @@ foreach($argv as $a) { $aggregator = new sspmod_statistics_Aggregator(TRUE); $aggregator->dumpConfig(); +$aggregator->debugInfo(); $results = $aggregator->aggregate($debug); +$aggregator->debugInfo(); if (!$dryrun) { $aggregator->store($results); diff --git a/modules/statistics/config-templates/module_statistics.php b/modules/statistics/config-templates/module_statistics.php index 66f452d7c30c297c2e72fbfa0bbb39fff8df7eec..f3482a3c236a0922d8b782e5606d9236273f2a82 100644 --- a/modules/statistics/config-templates/module_statistics.php +++ b/modules/statistics/config-templates/module_statistics.php @@ -47,116 +47,184 @@ $config = array ( * CGI timeout function. Both default to 300 seconds. */ 'time_limit' => 300, + + 'time_limit' => 300, - 'statrules' => array( - 'sso_hoursday' => array( - 'name' => 'SSO to service (per 15min)', - 'descr' => 'The number of logins at a Service Provider divided into slots of one hour. Each file contains data for one day (24 hours)', - - 'action' => 'saml20-sp-SSO', - 'col' => 6, // Service Provider EntityID - 'fieldPresentation' => array( - 'class' => 'statistics:Entity', - 'config' => 'saml20-sp-remote', - ), + 'timeres' => array( + 'day' => array( + 'name' => 'Day', 'slot' => 60*15, // Slots of 15 minutes 'fileslot' => 60*60*24, // One day (24 hours) file slots 'axislabelint' => 6*4, // Number of slots per label. 4 per hour *6 = 6 hours - + 'dateformat-period' => 'j. M', // 4. Mars + 'dateformat-intra' => 'j. M H:i', // 4. Mars 12:30 + ), + 'week' => array( + 'name' => 'Week', + 'slot' => 60*60, // Slots of one hour + 'fileslot' => 60*60*24*7, // 7 days of data in each file + 'axislabelint' => 24, // Number of slots per label. 24 is one each day 'dateformat-period' => 'j. M', // 4. Mars 'dateformat-intra' => 'j. M H:i', // 4. Mars 12:30 ), - 'sso_day80' => array( - 'name' => 'SSO to service (per day for 80 days)', - 'descr' => 'The number of logins at a Service Provider divided into slots of one day. Each file contains data for 80 days', + 'month' => array( + 'name' => 'Month', + 'slot' => 60*60*24, // Slots of one day + 'fileslot' => 60*60*24*30, // 30 days of data in each file + 'axislabelint' => 7, // Number of slots per label. 7 days => 1 week + 'dateformat-period' => 'j. M Y H:i', // 4. Mars 12:30 + 'dateformat-intra' => 'j. M', // 4. Mars + ), + 'monthaligned' => array( + 'name' => 'AlignedMonth', + 'slot' => 60*60*24, // Slots of one day + 'fileslot' => NULL, // 30 days of data in each file + 'customDateHandler' => 'month', + 'axislabelint' => 7, // Number of slots per label. 7 days => 1 week + 'dateformat-period' => 'j. M Y H:i', // 4. Mars 12:30 + 'dateformat-intra' => 'j. M', // 4. Mars + ), - 'action' => 'saml20-sp-SSO', - 'col' => 6, // Service Provider EntityID - 'fieldPresentation' => array( - 'class' => 'statistics:Entity', - 'config' => 'saml20-sp-remote', - ), + 'days180' => array( + 'name' => '180 days', 'slot' => 60*60*24, // Slots of 1 day (24 hours) - 'fileslot' => 60*60*24*80, // 80 days of data in each file + 'fileslot' => 60*60*24*180, // 80 days of data in each file + 'axislabelint' => 30, // Number of slots per label. 7 days => 1 week + 'dateformat-period' => 'j. M', // 4. Mars + 'dateformat-intra' => 'j. M', // 4. Mars + ), + ), + + 'time_limit' => 300, + + 'timeres' => array( + 'day' => array( + 'name' => 'Day', + 'slot' => 60*15, // Slots of 15 minutes + 'fileslot' => 60*60*24, // One day (24 hours) file slots + 'axislabelint' => 6*4, // Number of slots per label. 4 per hour *6 = 6 hours + 'dateformat-period' => 'j. M', // 4. Mars + 'dateformat-intra' => 'j. M H:i', // 4. Mars 12:30 + ), + 'week' => array( + 'name' => 'Week', + 'slot' => 60*60, // Slots of one hour + 'fileslot' => 60*60*24*7, // 7 days of data in each file + 'axislabelint' => 24, // Number of slots per label. 24 is one each day + 'dateformat-period' => 'j. M', // 4. Mars + 'dateformat-intra' => 'j. M H:i', // 4. Mars 12:30 + ), + 'month' => array( + 'name' => 'Month', + 'slot' => 60*60*24, // Slots of one day + 'fileslot' => 60*60*24*30, // 30 days of data in each file 'axislabelint' => 7, // Number of slots per label. 7 days => 1 week - + 'dateformat-period' => 'j. M Y H:i', // 4. Mars 12:30 + 'dateformat-intra' => 'j. M', // 4. Mars + ), + 'monthaligned' => array( + 'name' => 'AlignedMonth', + 'slot' => 60*60*24, // Slots of one day + 'fileslot' => NULL, // 30 days of data in each file + 'customDateHandler' => 'month', + 'axislabelint' => 7, // Number of slots per label. 7 days => 1 week + 'dateformat-period' => 'j. M Y H:i', // 4. Mars 12:30 + 'dateformat-intra' => 'j. M', // 4. Mars + ), + + 'days180' => array( + 'name' => '180 days', + 'slot' => 60*60*24, // Slots of 1 day (24 hours) + 'fileslot' => 60*60*24*180, // 80 days of data in each file + 'axislabelint' => 30, // Number of slots per label. 7 days => 1 week 'dateformat-period' => 'j. M', // 4. Mars 'dateformat-intra' => 'j. M', // 4. Mars ), - 'sso_day80realm' => array( - 'name' => 'SP by realm (per day for 80 days)', - 'descr' => 'The number of logins at a Service Provider divided into slots of one day. Each file contains data for 80 days', + ), + 'statrules' => array( + 'sloratio' => array( + 'name' => 'SSO to SLO ratio', + 'descr' => 'ratio', + 'type' => 'ratio', + 'action' => 'saml20-idp-SSO', - 'col' => 8, // Service Provider EntityID + 'col' => 6, // Service Provider EntityID 'fieldPresentation' => array( 'class' => 'statistics:Entity', 'config' => 'saml20-sp-remote', ), - 'slot' => 60*60*24, // Slots of 1 day (24 hours) - 'fileslot' => 60*60*24*80, // 80 days of data in each file - 'axislabelint' => 7, // Number of slots per label. 7 days => 1 week - - 'graph.total' => TRUE, - - - 'dateformat-period' => 'j. M', // 4. Mars - 'dateformat-intra' => 'j. M', // 4. Mars ), - 'sso_hoursweek' => array( - 'name' => 'SSO to service (per hour for a week)', - 'descr' => 'The number of logins at a Service Provider divided into slots of one hour. Each file contains data for one week.', - - 'action' => 'saml20-sp-SSO', + 'sso' => array( + 'name' => 'SSO to service', + 'descr' => 'The number of logins at a Service Provider.', + 'action' => 'saml20-idp-SSO', 'col' => 6, // Service Provider EntityID 'fieldPresentation' => array( 'class' => 'statistics:Entity', 'config' => 'saml20-sp-remote', ), - 'slot' => 60*60, // Slots of one hour - 'fileslot' => 60*60*24*7, // 7 days of data in each file - 'axislabelint' => 24, // Number of slots per label. 24 is one each day - - 'dateformat-period' => 'j. M', // 4. Mars - 'dateformat-intra' => 'j. M H:i', // 4. Mars 12:30 ), - 'sso_days' => array( - 'name' => 'SSO to service (per day for a month)', - 'descr' => 'The number of logins at a Service Provider divided into slots of one day. Each file contains data for 30 days.', - - 'action' => 'saml20-sp-SSO', + 'ssofirst' => array( + 'name' => 'SSO-first to service', + 'descr' => 'The number of logins at a Service Provider.', + 'action' => 'saml20-idp-SSO-first', 'col' => 6, // Service Provider EntityID 'fieldPresentation' => array( 'class' => 'statistics:Entity', 'config' => 'saml20-sp-remote', ), - 'slot' => 60*60*24, // Slots of one day - 'fileslot' => 60*60*24*30, // 30 days of data in each file - 'axislabelint' => 7, // Number of slots per label. 7 days => 1 week - - 'dateformat-period' => 'j. M Y H:i', // 4. Mars 12:30 - 'dateformat-intra' => 'j. M', // 4. Mars ), - 'slo_days' => array( - 'name' => 'Logout (per day for a month)', - 'descr' => 'The number of logouts divided into slots of one day. Each file contains data for 30 days.', - + 'ssoservicerealm' => array( + 'name' => 'SSO to service and realm', + 'descr' => 'The number of logins per realm and service provider.', + 'action' => 'saml20-idp-SSO', + 'col' => array(6,8), // Service Provider EntityID, realm + 'fieldPresentation' => array( + 'class' => 'feide:SPandOrg', + 'config' => 'saml20-sp-remote', + ), + ), + 'ssorealm' => array( + 'name' => 'SSO by realm', + 'descr' => 'The number of logins at a Service Provider divided into slots of one day. Each file contains data for 80 days', + 'action' => 'saml20-idp-SSO', + 'col' => 8, // Realm + 'fieldPresentation' => array( + 'class' => 'feide:Org', + 'config' => 'saml20-sp-remote', + ), + ), + 'slo' => array( + 'name' => 'Logout', + 'descr' => 'The number of initated Sinlge Logout.', 'action' => 'saml20-idp-SLO', 'col' => 7, // Service Provider EntityID that initiated the logout. 'fieldPresentation' => array( 'class' => 'statistics:Entity', 'config' => 'saml20-sp-remote', ), - 'slot' => 60*60*24, // Slots of one day - 'fileslot' => 60*60*24*30, // 30 days of data in each file - 'axislabelint' => 7, // Number of slots per label. 7 days => 1 week - - 'dateformat-period' => 'j. M Y H:i', // 4. Mars 12:30 - 'dateformat-intra' => 'j. M', // 4. Mars ), - - ), + 'consent' => array( + 'name' => 'Consent', + 'descr' => 'Consent statistics. Everytime a user logs in to a service an entry is logged for one of three states: consent was found, consent was not found or consent storage was not available.', + 'action' => 'consent', + 'col' => 6, + 'fieldPresentation' => array( + 'class' => 'statistics:Entity', + 'config' => 'saml20-sp-remote', + ), + ), + 'consentresponse' => array( + 'name' => 'Consent response', + 'descr' => 'Consent response statistics. Everytime a user accepts consent, it is logged whether the user selected to remember the consent to next time.', + 'action' => 'consentResponse', + 'col' => 6, + 'fieldPresentation' => array( + 'class' => 'statistics:Entity', + 'config' => 'saml20-sp-remote', + ), + ), ); -?> \ No newline at end of file diff --git a/modules/statistics/hooks/hook_frontpage.php b/modules/statistics/hooks/hook_frontpage.php index 4b5596b4b70d58ebeba7c8b89bbdebfc69cd7500..ac58b8237aab6dd9cdcab11328c710afbd45c102 100644 --- a/modules/statistics/hooks/hook_frontpage.php +++ b/modules/statistics/hooks/hook_frontpage.php @@ -13,6 +13,11 @@ function statistics_hook_frontpage(&$links) { 'text' => array('en' => 'Show statistics', 'no' => 'Vis statistikk'), 'shorttext' => array('en' => 'Statistics', 'no' => 'Statistikk'), ); + $links['links']['statisticsmeta'] = array( + 'href' => SimpleSAML_Module::getModuleURL('statistics/statmeta.php'), + 'text' => array('en' => 'Show statistics metadata', 'no' => 'Vis statistikk metadata'), + 'shorttext' => array('en' => 'Statistics metadata', 'no' => 'Statistikk metadata'), + ); } ?> \ No newline at end of file diff --git a/modules/statistics/lib/Aggregator.php b/modules/statistics/lib/Aggregator.php index 46e41f365b84479748c794d5d38c9d0f3df25a0f..1a82ef9d6bba39b0081878e9feec5d6697312e6c 100644 --- a/modules/statistics/lib/Aggregator.php +++ b/modules/statistics/lib/Aggregator.php @@ -11,8 +11,10 @@ class sspmod_statistics_Aggregator { private $inputfile; private $statrules; private $offset; - + private $metadata; private $fromcmdline; + + private $starttime; /** * Constructor @@ -25,37 +27,75 @@ class sspmod_statistics_Aggregator { $this->statdir = $this->statconfig->getValue('statdir'); $this->inputfile = $this->statconfig->getValue('inputfile'); $this->statrules = $this->statconfig->getValue('statrules'); + $this->timeres = $this->statconfig->getValue('timeres'); $this->offset = $this->statconfig->getValue('offset', 0); + $this->metadata = NULL; + + $this->starttime = time(); } public function dumpConfig() { - echo 'Statistics directory : ' . $this->statdir . "\n"; echo 'Input file : ' . $this->inputfile . "\n"; echo 'Offset : ' . $this->offset . "\n"; + } + + public function debugInfo() { + echo 'Memory usage : ' . number_format(memory_get_usage() / (1024*1024), 2) . " MB\n"; + } + + public function loadMetadata() { + $filename = $this->statdir . '/.stat.metadata'; + $metadata = NULL; + if (file_exists($filename)) { + $metadata = unserialize(file_get_contents($filename)); + } + $this->metadata = $metadata; + } + + public function getMetadata() { + return $this->metadata; + } + + public function saveMetadata() { + $this->metadata['time'] = time() - $this->starttime; + $this->metadata['memory'] = memory_get_usage(); + $this->metadata['lastrun'] = time(); + $filename = $this->statdir . '/.stat.metadata'; + file_put_contents($filename, serialize($this->metadata), LOCK_EX); } - - public function aggregate($debug = FALSE) { + $this->loadMetadata(); + if (!is_dir($this->statdir)) throw new Exception('Statistics module: output dir do not exists [' . $this->statdir . ']'); if (!file_exists($this->inputfile)) throw new Exception('Statistics module: input file do not exists [' . $this->inputfile . ']'); - $file = fopen($this->inputfile, 'r'); #$logfile = file($this->inputfile, FILE_IGNORE_NEW_LINES ); - $logparser = new sspmod_statistics_LogParser( $this->statconfig->getValue('datestart', 0), $this->statconfig->getValue('datelength', 15), $this->statconfig->getValue('offsetspan', 44) ); - $datehandler = new sspmod_statistics_DateHandler($this->offset); + $datehandler = array( + 'default' => new sspmod_statistics_DateHandler($this->offset), + 'month' => new sspmod_statistics_DateHandlerMonth($this->offset), + ); + + $notBefore = 0; $lastRead = 0; $lastlinehash = '-'; + if (isset($this->metadata)) { + $notBefore = $this->metadata['notBefore']; + $lastlinehash = $this->metadata['lastlinehash']; + } + + $lastlogline = 'sdfsdf'; + $lastlineflip = FALSE; $results = array(); $i = 0; @@ -66,7 +106,7 @@ class sspmod_statistics_Aggregator { // Continue if STAT is not found on line. if (!preg_match('/STAT/', $logline)) continue; - $i++; + $i++; $lastlogline = $logline; // Parse log, and extract epoch time and rest of content. $epoch = $logparser->parseEpoch($logline); @@ -76,11 +116,8 @@ class sspmod_statistics_Aggregator { if ($this->fromcmdline && ($i % 10000) == 0) { echo("Read line " . $i . "\n"); } - if ($debug) { - - echo("----------------------------------------\n"); echo('Log line: ' . $logline . "\n"); echo('Date parse [' . substr($logline, 0, $this->statconfig->getValue('datelength', 15)) . '] to [' . date(DATE_RFC822, $epoch) . ']' . "\n"); @@ -88,84 +125,165 @@ class sspmod_statistics_Aggregator { if ($i >= 13) exit; } + if ($epoch > $lastRead) $lastRead = $epoch; + if ($epoch === $notBefore) { + if(!$lastlineflip) { + if (sha1($logline) === $lastlinehash) { + $lastlineflip = TRUE; + } + continue; + } + } + if ($epoch < $notBefore) continue; // Iterate all the statrules from config. foreach ($this->statrules AS $rulename => $rule) { - // echo 'Comparing action: [' . $rule['action'] . '] with [' . $action . ']' . "\n"; + foreach($this->timeres AS $tres => $tresconfig ) { - $timeslot = $datehandler->toSlot($epoch, $rule['slot']); - $fileslot = $datehandler->toSlot($epoch, $rule['fileslot']); //print_r($content); - - if (isset($rule['action']) && ($action !== $rule['action'])) continue; + // echo 'Comparing action: [' . $rule['action'] . '] with [' . $action . ']' . "\n"; + $dh = 'default'; + if (isset($tresconfig['customDateHandler'])) $dh = $tresconfig['customDateHandler']; + + $timeslot = $datehandler['default']->toSlot($epoch, $tresconfig['slot']); + $fileslot = $datehandler[$dh]->toSlot($epoch, $tresconfig['fileslot']); //print_r($content); + if (isset($rule['action']) && ($action !== $rule['action'])) continue; - $difcol = trim($content[$rule['col']]); // echo '[...' . $difcol . '...]'; + #$difcol = trim($content[$rule['col']]); // echo '[...' . $difcol . '...]'; + $difcol = self::getDifCol($content, $rule['col']); - if (!isset($results[$rulename][$fileslot][$timeslot]['_'])) $results[$rulename][$fileslot][$timeslot]['_'] = 0; - if (!isset($results[$rulename][$fileslot][$timeslot][$difcol])) $results[$rulename][$fileslot][$timeslot][$difcol] = 0; + if (!isset($results[$rulename][$tres][$fileslot][$timeslot]['_'])) $results[$rulename][$tres][$fileslot][$timeslot]['_'] = 0; + if (!isset($results[$rulename][$tres][$fileslot][$timeslot][$difcol])) $results[$rulename][$tres][$fileslot][$timeslot][$difcol] = 0; - $results[$rulename][$fileslot][$timeslot]['_']++; - $results[$rulename][$fileslot][$timeslot][$difcol]++; - + $results[$rulename][$tres][$fileslot][$timeslot]['_']++; + $results[$rulename][$tres][$fileslot][$timeslot][$difcol]++; + } + } + } + $this->metadata['notBefore'] = $lastRead; + $this->metadata['lastline'] = $lastlogline; + $this->metadata['lastlinehash'] = sha1($lastlogline); + return $results; + } + + private static function getDifCol($content, $colrule) { + if (is_string($colrule)) { + return trim($content[$colrule]); + } elseif(is_array($colrule)) { + $difcols = array(); + foreach($colrule AS $cr) { + $difcols[] = trim($content[$cr]); + } + return join('|', $difcols); + } else { + return 'NA'; + } + } + + private function cummulateData($previous, $newdata) { + $dataset = array(); + foreach($previous AS $slot => $dataarray) { + if (!array_key_exists($slot, $dataset)) $dataset[$slot] = array(); + foreach($dataarray AS $key => $data) { + if (!array_key_exists($key, $dataset[$slot])) $dataset[$slot][$key] = 0; + $dataset[$slot][$key] += $data; + } + } + foreach($newdata AS $slot => $dataarray) { + if (!array_key_exists($slot, $dataset)) $dataset[$slot] = array(); + foreach($dataarray AS $key => $data) { + if (!array_key_exists($key, $dataset[$slot])) $dataset[$slot][$key] = 0; + $dataset[$slot][$key] += $data; } } - return $results; + return $dataset; } public function store($results) { - $datehandler = new sspmod_statistics_DateHandler($this->offset); + // print_r($results); // exit; + + $datehandler = array( + 'default' => new sspmod_statistics_DateHandler($this->offset), + 'month' => new sspmod_statistics_DateHandlerMonth($this->offset), + ); // Iterate the first level of results, which is per rule, as defined in the config. - foreach ($results AS $rulename => $ruleresults) { + foreach ($results AS $rulename => $timeresdata) { + // $timeresl = array_keys($timeresdata); + // + // print_r($timeresl); exit; - $filenos = array_keys($ruleresults); - $lastfile = $filenos[count($filenos)-1]; + // Iterate over time resolutions + foreach($timeresdata AS $tres => $resres) { + + $dh = 'default'; + if (isset($this->timeres[$tres]['customDateHandler'])) $dh = $this->timeres[$tres]['customDateHandler']; - // Iterate the second level of results, which is the fileslot. - foreach ($ruleresults AS $fileno => $fileres) { + $filenos = array_keys($resres); + $lastfile = $filenos[count($filenos)-1]; - $slotlist = array_keys($fileres); + // Iterate the second level of results, which is the fileslot. + foreach ($resres AS $fileno => $fileres) { + + + // Slots that have data. + $slotlist = array_keys($fileres); - $maxslot = $slotlist[count($slotlist)-1]; - #print_r($slotlist); - - // Get start and end slot number within the file, based on the fileslot. - $start = (int)$datehandler->toSlot($datehandler->fromSlot($fileno, $this->statrules[$rulename]['fileslot']), $this->statrules[$rulename]['slot']); - $end = (int)$datehandler->toSlot($datehandler->fromSlot($fileno+1, $this->statrules[$rulename]['fileslot']), $this->statrules[$rulename]['slot']); - - // Fill in missing entries and sort file results - $filledresult = array(); - for ($slot = $start; $slot < $end; $slot++) { - #print_r(gettype($slot)); - if (array_key_exists($slot, $fileres)) { - $filledresult[$slot] = $fileres[$slot]; - } else { - #echo('SLot [' . $slot . '] of [' . $maxslot . ']' . "\n"); - if ($lastfile == $fileno && $slot > $maxslot) { - #if ($slot > $maxslot) { - $filledresult[$slot] = array('_' => NULL); + // The last slot. + $maxslot = $slotlist[count($slotlist)-1]; + #print_r($slotlist); + + // Get start and end slot number within the file, based on the fileslot. + $start = (int)$datehandler['default']->toSlot( + $datehandler[$dh]->fromSlot($fileno, $this->timeres[$tres]['fileslot']), + $this->timeres[$tres]['slot']); + $end = (int)$datehandler['default']->toSlot( + $datehandler[$dh]->fromSlot($fileno+1, $this->timeres[$tres]['fileslot']), + $this->timeres[$tres]['slot']); + + // echo('from slot ' . $start . ' to slot ' . $end . ' maxslot ' . $maxslot . "\n"); + // print_r($slotlist); + // exit; + + // Fill in missing entries and sort file results + $filledresult = array(); + for ($slot = $start; $slot < $end; $slot++) { + if (array_key_exists($slot, $fileres)) { + $filledresult[$slot] = $fileres[$slot]; } else { - $filledresult[$slot] = array('_' => 0); - } + #echo('SLot [' . $slot . '] of [' . $maxslot . ']' . "\n"); + if ($lastfile == $fileno && $slot > $maxslot) { + $filledresult[$slot] = array('_' => NULL); + } else { + $filledresult[$slot] = array('_' => 0); + } + } + # print_r($filledresult[$slot]); + # = (isset($fileres[$slot])) ? $fileres[$slot] : array('_' => NULL); + } + // print_r($filledresult); exit; + + $filename = $this->statdir . '/' . $rulename . '-' . $tres . '-' . $fileno . '.stat'; + if (file_exists($filename)) { + echo('Reading existing file: ' . $filename . "\n"); + $previousData = unserialize(file_get_contents($filename)); + $filledresult = $this->cummulateData($previousData, $filledresult); } - #print_r($filledresult[$slot]); -# = (isset($fileres[$slot])) ? $fileres[$slot] : array('_' => NULL); - } - #print_r($filledresult); exit; + // store file + echo('Writing to file: ' . $filename . "\n"); + file_put_contents($filename, serialize($filledresult), LOCK_EX); + } - // store file - file_put_contents($this->statdir . '/' . $rulename . '-' . $fileno . '.stat', serialize($filledresult), LOCK_EX ); } + } + $this->saveMetadata(); } - } - -?> \ No newline at end of file diff --git a/modules/statistics/lib/DateHandler.php b/modules/statistics/lib/DateHandler.php index ebc07a11eaa349bbeefd87b7aae89835f8795dce..acfe70553aede0e6fa40b7910a304f2ad6ca0cf9 100644 --- a/modules/statistics/lib/DateHandler.php +++ b/modules/statistics/lib/DateHandler.php @@ -15,11 +15,9 @@ class sspmod_statistics_DateHandler { */ public function __construct($offset) { $this->offset = $offset; - - } - private function getDST($timestamp) { + protected function getDST($timestamp) { if (idate('I', $timestamp)) return 3600; return 0; } @@ -30,6 +28,8 @@ class sspmod_statistics_DateHandler { } public function fromSlot($slot, $slotsize) { + // echo("slot $slot slotsize $slotsize offset " . $this->offset); + // throw new Exception(); $temp = $slot*$slotsize - $this->offset; $dst = $this->getDST($temp); return $slot*$slotsize - $this->offset - $dst; @@ -43,6 +43,13 @@ class sspmod_statistics_DateHandler { return $this->prettyDateEpoch($this->fromSlot($slot, $slotsize), $dateformat); } + + public function prettyHeader($from, $to, $slotsize, $dateformat) { + $text = $this->prettyDateSlot($from, $slotsize, $dateformat); + $text .= ' to '; + $text .= $this->prettyDateSlot($to, $slotsize, $dateformat); + return $text; + } } @@ -56,4 +63,3 @@ class sspmod_statistics_DateHandler { // print_r($timestamp); // print_r($restcols); if ($i++ > 5) exit; -?> \ No newline at end of file diff --git a/modules/statistics/lib/DateHandlerMonth.php b/modules/statistics/lib/DateHandlerMonth.php new file mode 100644 index 0000000000000000000000000000000000000000..945f3fa179b4cbccbb03db72ea635685b40bd2a6 --- /dev/null +++ b/modules/statistics/lib/DateHandlerMonth.php @@ -0,0 +1,60 @@ +<?php +/* + * @author Andreas Åkre Solberg <andreas.solberg@uninett.no> + * @package simpleSAMLphp + * @version $Id$ + */ +class sspmod_statistics_DateHandlerMonth extends sspmod_statistics_DateHandler { + + + + /** + * Constructor + * + * @param array $offset Date offset + */ + public function __construct($offset) { + $this->offset = $offset; + } + + + public function toSlot($epoch, $slotsize) { + $dsttime = $this->getDST($epoch) + $epoch; + $parsed = getdate($dsttime); + // print_r($parsed); + $slot = (($parsed['year'] - 2000) * 12) + $parsed['mon'] - 1; + // echo('converting ' . $epoch . ' to ' . $slot ); exit; + return $slot; + } + + public function fromSlot($slot, $slotsize) { + + $month = ($slot % 12); + $year = 2000 + floor($slot / 12); + + $epoch = mktime(0, 0, 0, $month + 1, 1, $year, FALSE); + // echo('epoch ' . $epoch . ' from slot '. $slot . " year " . $year . " month " . $month . "\n"); + return $epoch; + } + + public function prettyHeader($from, $to, $slotsize, $dateformat) { + + $month = ($from % 12) + 1; + $year = 2000 + floor($from / 12); + + return $year . '-' . $month; + } + + +} + +// $datestr = substr($logline,0,$datenumbers); +// #$datestr = substr($logline,0,23); +// $timestamp = parse15($datestr) + $offset; +// $restofline = substr($logline,$datenumbers+1); +// $restcols = split(' ', $restofline); +// $action = $restcols[5]; + +// print_r($timestamp); +// print_r($restcols); if ($i++ > 5) exit; + diff --git a/modules/statistics/lib/Graph/GoogleCharts.php b/modules/statistics/lib/Graph/GoogleCharts.php index 4eec87d593330b358dd944644a274b11a80a5537..3525e626cd09d66101e6d9b428d92e852b6abcee 100644 --- a/modules/statistics/lib/Graph/GoogleCharts.php +++ b/modules/statistics/lib/Graph/GoogleCharts.php @@ -92,7 +92,6 @@ class sspmod_statistics_Graph_GoogleCharts { public function showPie($axis, $datasets) { - $url = 'http://chart.apis.google.com/chart?' . // Dimension of graph. Default is 800x350 @@ -105,7 +104,6 @@ class sspmod_statistics_Graph_GoogleCharts { '&cht=p' . '&chl=' . $this->encodeaxis($axis); - return $url; } diff --git a/modules/statistics/lib/Ruleset.php b/modules/statistics/lib/Ruleset.php new file mode 100644 index 0000000000000000000000000000000000000000..24fe6c2f0be203f55f11f38a6abff8bee763ea2e --- /dev/null +++ b/modules/statistics/lib/Ruleset.php @@ -0,0 +1,89 @@ +<?php +/* + * @author Andreas Åkre Solberg <andreas.solberg@uninett.no> + * @package simpleSAMLphp + * @version $Id$ + */ +class sspmod_statistics_Ruleset { + + private $statconfig; + private $availrulenames; + private $availrules; + private $available; + + /** + * Constructor + */ + public function __construct($statconfig) { + $this->statconfig = $statconfig; + $this->init(); + } + + private function init() { + + $statdir = $this->statconfig->getValue('statdir'); + $inputfile = $this->statconfig->getValue('inputfile'); + $statrules = $this->statconfig->getValue('statrules'); + $timeres = $this->statconfig->getValue('timeres'); + + /* + * Walk through file lists, and get available [rule][fileslot]... + */ + if (!is_dir($statdir)) + throw new Exception('Statisics output directory [' . $statdir . '] does not exists.'); + $filelist = scandir($statdir); + $this->available = array(); + foreach ($filelist AS $file) { + if (preg_match('/([a-z0-9_]+)-([a-z0-9_]+)-([0-9]+)\.stat/', $file, $matches)) { + if (array_key_exists($matches[1], $statrules)) { + if (array_key_exists($matches[2], $timeres)) + $this->available[$matches[1]][$matches[2]][] = $matches[3]; + } + } + } + if (empty($this->available)) + throw new Exception('No aggregated statistics files found in [' . $statdir . ']'); + + /* + * Create array with information about available rules.. + */ + $available_rules = array(); + foreach ($this->available AS $key => $av) { + $available_rules[$key] = array('name' => $statrules[$key]['name'], 'descr' => $statrules[$key]['descr']); + } + $this->availrules = array_keys($available_rules); + $this->availrulenames = $available_rules; + + } + + public function availableRules() { + return $this->availrules; + } + + public function availableRulesNames() { + return $this->availrulenames; + } + + /** + * Resolve which rule is selected. Taking user preference and checks if it exists. + */ + private function resolveSelectedRule($preferRule = NULL) { + $rule = $this->statconfig->getString('default', $this->availrules[0]); + if(!empty($preferRule)) { + if (in_array($preferRule, $this->availrules)) { + $rule = $preferRule; + } + } + return $rule; + } + + public function getRule($preferRule) { + $rule = $this->resolveSelectedRule($preferRule); + $statrulesConfig = $this->statconfig->getConfigItem('statrules'); + $statruleConfig = $statrulesConfig->getConfigItem($rule); + $statrule = new sspmod_statistics_StatRule($this->statconfig, $statruleConfig, $rule, $this->available[$rule]); + return $statrule; + } + +} + diff --git a/modules/statistics/lib/StatDataset.php b/modules/statistics/lib/StatDataset.php new file mode 100644 index 0000000000000000000000000000000000000000..e1eccfd802128ee3fd21cb9bff86580e90d5ffe5 --- /dev/null +++ b/modules/statistics/lib/StatDataset.php @@ -0,0 +1,313 @@ +<?php +/* + * @author Andreas Åkre Solberg <andreas.solberg@uninett.no> + * @package simpleSAMLphp + * @version $Id$ + */ +class sspmod_statistics_StatDataset { + + private $statconfig; + private $ruleconfig; + private $timeresconfig; + private $ruleid; + + private $fileslot; + private $timeres; + + private $delimiter; + private $results; + private $summary; + private $max; + + private $datehandlerFile; + private $datehandlerTick; + + /** + * Constructor + */ + public function __construct($statconfig, $ruleconfig, $ruleid, $timeres, $fileslot) { + assert('$statconfig instanceof SimpleSAML_Configuration'); + assert('$ruleconfig instanceof SimpleSAML_Configuration'); + $this->statconfig = $statconfig; + $this->ruleconfig = $ruleconfig; + + $timeresconfigs = $statconfig->getConfigItem('timeres'); + $this->timeresconfig = $timeresconfigs->getConfigItem($timeres); + + $this->ruleid = $ruleid; + $this->fileslot = $fileslot; + $this->timeres = $timeres; + + + $this->delimiter = '_'; + $this->max = 0; + + $this->datehandlerTick = new sspmod_statistics_DateHandler($this->statconfig->getValue('offset', 0)); + if ($this->timeresconfig->getValue('customDateHandler', 'default') === 'month') { + $this->datehandlerFile = new sspmod_statistics_DateHandlerMonth(0); + } else { + $datehandlerFile = $this->datehandlerTick; + } + + + $this->loadData(); + + + } + + public function getFileSlot() { + return $this->fileslot; + } + + public function getTimeRes() { + return $this->timeres; + } + + public function setDelimiter($delimiter = '_') { + if (empty($delimiter)) $delimiter = '_'; + $this->delimiter = $delimiter; + // echo 'delimiter set to ' . $delimiter; exit; + } + public function getDelimiter() { + if ($this->delimiter === '_') return NULL; + return $this->delimiter; + } + + public function calculateMax() { + + /* + * Get rule specific configuration from the configuration file. + */ + $slotsize = $this->ruleconfig->getValue('slot'); + $dateformat_period = $this->timeresconfig->getValue('dateformat-period'); + $dateformat_intra = $this->timeresconfig->getValue('dateformat-intra'); + // $axislabelint = $this->ruleconfig->getValue('axislabelint'); + + + $maxvalue = 0; $maxvaluetime = NULL; + foreach($this->results AS $slot => &$res) { + if (!array_key_exists($this->delimiter, $res)) $res[$this->delimiter] = 0; + if ($res[$this->delimiter] > $maxvalue) { + $maxvaluetime = $this->datehandlerTick->prettyDateSlot($slot, $slotsize, $dateformat_intra); + } + $maxvalue = max($res[$this->delimiter],$maxvalue); + } + $this->max = sspmod_statistics_Graph_GoogleCharts::roof($maxvalue); + } + + public function getDebugData() { + $debugdata = array(); + + $slotsize = $this->timeresconfig->getValue('slot'); + $dateformat_period = $this->timeresconfig->getValue('dateformat-period'); + $dateformat_intra = $this->timeresconfig->getValue('dateformat-intra'); + // $axislabelint = $this->ruleconfig->getValue('axislabelint'); + + foreach($this->results AS $slot => &$res) { + $debugdata[$slot] = array($this->datehandlerTick->prettyDateSlot($slot, $slotsize, $dateformat_intra), $res[$this->delimiter] ); + } + return $debugdata; + } + + public function aggregateSummary() { + + /** + * Aggregate summary table from dataset. To be used in the table view. + */ + $this->summary = array(); + foreach($this->results AS $slot => $res) { + foreach ($res AS $key => $value) { + if (array_key_exists($key, $this->summary)) { + $this->summary[$key] += $value; + } else { + $this->summary[$key] = $value; + } + } + } + asort($this->summary); + $this->summary = array_reverse($this->summary, TRUE); + // echo '<pre>'; print_r($summaryDataset); exit; + } + + public function getTopDelimiters() { + /* + * Create a list of delimiter keys that has the highest total summary in this period. + */ + $topdelimiters = array(); + $maxdelimiters = 4; $i = 0; + foreach($this->summary AS $key => $value) { + if ($key !== '_') + $topdelimiters[] = $key; + if ($i++ >= $maxdelimiters) break; + } + return $topdelimiters; + + } + + public function availDelimiters() { + $availDelimiters = array(); + foreach($this->summary AS $key => $value) { + if ($key !== '_') + $topdelimiters[] = $key; + $availdelimiters[$key] = 1; + } + return array_keys($availdelimiters); + } + + public function getPieData() { + + $piedata = array(); $sum = 0; + $topdelimiters = $this->getTopDelimiters(); + + foreach($topdelimiters AS $td) { + $sum += $this->summary[$td]; + $piedata[] = number_format(100*$this->summary[$td] / $this->summary['_'], 2); + } + $piedata[] = number_format(100 - 100*($sum /$this->summary['_']), 2); + return $piedata; + } + + public function getMax() { + return $this->max; + } + + public function getSummary() { + return $this->summary; + } + + public function getResults() { + return $this->results; + } + + public function getAxis() { + $slotsize = $this->timeresconfig->getValue('slot'); + $dateformat_period = $this->timeresconfig->getValue('dateformat-period'); + $dateformat_intra = $this->timeresconfig->getValue('dateformat-intra'); + $axislabelint = $this->timeresconfig->getValue('axislabelint'); + + + $axis = array(); + $axispos = array(); + $xentries = count($this->results); + $lastslot = 0; $i = 0; + + foreach($this->results AS $slot => $res) { + + // check if there should be an axis here... + if ( $slot % $axislabelint == 0) { + $axis[] = $this->datehandlerTick->prettyDateSlot($slot, $slotsize, $dateformat_intra); + $axispos[] = (($i)/($xentries-1)); + // echo 'set axis on [' . $slot . '] = [' . $datehandler->prettyDateSlot($slot, $slotsize, $dateformat_intra) . ']'; + } + $lastslot = $slot; + $i++; + } + + $axis[] = $this->datehandlerTick->prettyDateSlot($lastslot+1, $slotsize, $dateformat_intra); + + return array('axis' => $axis, 'axispos' => $axispos); + } + + + /* + * Walk through dataset to get percent values from max into dataset[]. + */ + public function getPercentValues() { + + + $slotsize = $this->timeresconfig->getValue('slot'); + $dateformat_period = $this->timeresconfig->getValue('dateformat-period'); + $dateformat_intra = $this->timeresconfig->getValue('dateformat-intra'); + $axislabelint = $this->timeresconfig->getValue('axislabelint'); + + #$max = 25; + $xentries = count($this->results); + $lastslot = 0; $i = 0; + + + $dataset = array(); + foreach($this->results AS $slot => $res) { + #echo ('<p>new value: ' . number_format(100*$res[$delimiter] / $max, 2)); + // echo('<hr><p>delimiter [<tt>' .$delimiter . '</tt>].'); + // echo('<p>Res <pre>'); print_r($res); echo( '</pre>'); + // echo('<p>return <pre>'); print_r(isset($res[$delimiter]) ? $res[$delimiter] : 'NO'); echo('</pre>'); + if (array_key_exists($this->delimiter, $res)) { + if ($res[$this->delimiter] === NULL) { + $dataset[] = -1; + } else { + $dataset[] = number_format(100*$res[$this->delimiter] / $this->max, 2); + } + } else { + $dataset[] = '0'; + } + // foreach(array_keys($res) AS $nd) $availdelimiters[$nd] = 1; + + $lastslot = $slot; + $i++; + } + + return $dataset; + } + + + + public function getDelimiterPresentation() { + $config = SimpleSAML_Configuration::getInstance(); + $t = new SimpleSAML_XHTML_Template($config, 'statistics:statistics-tpl.php'); + + $availdelimiters = $this->availDelimiters(); + + + + /* + * Create a delimiter presentation filter for this rule... + */ + if ($this->ruleconfig->hasValue('fieldPresentation')) { + $fieldpresConfig = $this->ruleconfig->getConfigItem('fieldPresentation'); + $classname = SimpleSAML_Module::resolveClass($fieldpresConfig->getValue('class'), 'Statistics_FieldPresentation'); + if (!class_exists($classname)) + throw new Exception('Could not find field presentation plugin [' . $classname . ']: No class found'); + $presentationHandler = new $classname($availdelimiters, $fieldpresConfig->getValue('config'), $t); + + return $presentationHandler->getPresentation(); + } + + return NULL; + } + + public function getDelimiterPresentationPie() { + $topdelimiters = $this->getTopDelimiters(); + $delimiterPresentation = $this->getDelimiterPresentation(); + + $pieaxis = array(); + foreach($topdelimiters AS $key) { + $keyName = $key; + if(array_key_exists($key, $delimiterPresentation)) $keyName = $delimiterPresentation[$key]; + $pieaxis[] = $keyName; + } + $pieaxis[] = 'Others'; + return $pieaxis; + } + + + + public function loadData() { + + $statdir = $this->statconfig->getValue('statdir'); + + // Get file and extract results. + $resultFileName = $statdir . '/' . $this->ruleid . '-' . $this->timeres . '-'. $this->fileslot . '.stat'; + if (!file_exists($resultFileName)) + throw new Exception('Aggregated statitics file [' . $resultFileName . '] not found.'); + if (!is_readable($resultFileName)) + throw new Exception('Could not read statitics file [' . $resultFileName . ']. Bad file permissions?'); + $resultfile = file_get_contents($resultFileName); + $this->results = unserialize($resultfile); + if (empty($this->results)) + throw new Exception('Aggregated statistics in file [' . $resultFileName . '] was empty.'); + + // echo('<pre>'); print_r($this->results); exit; + } + +} + diff --git a/modules/statistics/lib/StatRule.php b/modules/statistics/lib/StatRule.php new file mode 100644 index 0000000000000000000000000000000000000000..c5e17b4f893ba14929e1afd8ce4d27babd991e66 --- /dev/null +++ b/modules/statistics/lib/StatRule.php @@ -0,0 +1,119 @@ +<?php +/* + * @author Andreas Åkre Solberg <andreas.solberg@uninett.no> + * @package simpleSAMLphp + * @version $Id$ + */ +class sspmod_statistics_StatRule { + + private $statconfig; + private $ruleconfig; + private $ruleid; + private $available; + // private $datehandler; + /** + * Constructor + */ + public function __construct($statconfig, $ruleconfig, $ruleid, $available) { + assert('$statconfig instanceof SimpleSAML_Configuration'); + assert('$ruleconfig instanceof SimpleSAML_Configuration'); + $this->statconfig = $statconfig; + $this->ruleconfig = $ruleconfig; + $this->ruleid = $ruleid; + $this->available = $available; + + // $this->datehandlerFile = $datehandler; + // $this->datehandlerTick = $datehandler; + } + + public function getRuleID() { + return $this->ruleid; + } + + public function init() { + + } + + public function availableTimeRes() { + $timeresConfigs = $this->statconfig->getValue('timeres'); + $available_times = array(); + foreach ($timeresConfigs AS $tres => $tresconfig) { + if (array_key_exists($tres, $this->available)) + $available_times[$tres] = $tresconfig['name']; + } + // echo('<pre>'); print_r($available_times); exit; + return $available_times; + } + + + public function availableFileSlots($timeres) { + $timeresConfigs = $this->statconfig->getValue('timeres'); + $timeresConfig = $timeresConfigs[$timeres]; + + if (isset($timeresConfig['customDateHandler']) && $timeresConfig['customDateHandler'] == 'month') { + $datehandler = new sspmod_statistics_DateHandlerMonth(0); + } else { + $datehandler = new sspmod_statistics_DateHandler($this->statconfig->getValue('offset', 0)); + } + + + /* + * Get list of avaiable times in current file (rule) + */ + $available_times = array(); + foreach ($this->available[$timeres] AS $slot) { + $available_times[$slot] = $datehandler->prettyHeader($slot, $slot+1, $timeresConfig['fileslot'], $timeresConfig['dateformat-period']); + } + return $available_times; + } + + private function resolveTimeRes($preferTimeRes) { + $timeresavailable = array_keys($this->available); + $timeres = $timeresavailable[0]; + + // Then check if the user have provided one that is valid. + if (in_array($preferTimeRes, $timeresavailable)) { + $timeres = $preferTimeRes; + } + return $timeres; + } + + private function resolveFileSlot($timeres, $preferTime) { + + // Get which time (fileslot) to use.. First get a default, which is the most recent one. + $fileslot = $this->available[$timeres][count($this->available[$timeres])-1]; + // Then check if the user have provided one. + if (in_array($preferTime, $this->available[$timeres])) { + $fileslot = $preferTime; + } + return $fileslot; + } + + + public function getTimeNavigation($timeres, $preferTime) { + $fileslot = $this->resolveFileSlot($timeres, $preferTime); + + // Extract previous and next time slots... + $available_times_prev = NULL; $available_times_next = NULL; + + $timeslots = array_values($this->available[$timeres]); + sort($timeslots, SORT_NUMERIC); + $timeslotindex = array_flip($timeslots); + + if ($timeslotindex[$fileslot] > 0) + $available_times_prev = $timeslots[$timeslotindex[$fileslot]-1]; + if ($timeslotindex[$fileslot] < (count($timeslotindex)-1) ) + $available_times_next = $timeslots[$timeslotindex[$fileslot]+1]; + return array('prev' => $available_times_prev, 'next' => $available_times_next); + } + + public function getDataSet($preferTimeRes, $preferTime) { + $timeres = $this->resolveTimeRes($preferTimeRes); + $fileslot = $this->resolveFileSlot($timeres, $preferTime); + $dataset = new sspmod_statistics_StatDataset($this->statconfig, $this->ruleconfig, $this->ruleid, $timeres, $fileslot); + return $dataset; + } + + +} + diff --git a/modules/statistics/templates/statistics-tpl.php b/modules/statistics/templates/statistics-tpl.php index 48b4c6c71fbeb1a8cd2727bfe28941e66d37cd3c..47b6114ed0fbdfd0cd829c6d6771af1a88e5042a 100644 --- a/modules/statistics/templates/statistics-tpl.php +++ b/modules/statistics/templates/statistics-tpl.php @@ -3,7 +3,7 @@ $this->data['header'] = 'SimpleSAMLphp Statistics'; $this->data['jquery'] = array('version' => '1.6', 'core' => TRUE, 'ui' => TRUE, 'css' => TRUE); -$this->data['hideLanguageBar'] = TRUE; +// $this->data['hideLanguageBar'] = TRUE; $this->data['head'] =''; $this->data['head'] .= '<script type="text/javascript"> @@ -12,9 +12,36 @@ $(document).ready(function() { }); </script>'; - $this->includeAtTemplateBase('includes/header.php'); + +function getBaseURL($t, $type = 'get', $key = NULL, $value = NULL) { + $vars = array( + 'rule' => $t->data['selected.rule'], + 'time' => $t->data['selected.time'], + 'res' => $t->data['selected.timeres'], + ); + if (isset($t->data['selected.delimiter'])) $vars['d'] = $t->data['selected.delimiter']; + + if (isset($key)) { + if (isset($vars[$key])) unset($vars[$key]); + if (isset($value)) $vars[$key] = $value; + } + + if ($type === 'get') { + return 'showstats.php?' . http_build_query($vars, '', '&'); + } else { + $text = ''; + foreach($vars AS $k => $v) { + $text .= '<input type="hidden" name="' . $k . '" value="'. htmlspecialchars($v) . '" />' . "\n"; + } + return $text; + } + +} + + + ?> <style type="text/css" media="all"> @@ -64,7 +91,9 @@ td.datacontent { echo('<h1>'. $this->data['available.rules'][$this->data['selected.rule']]['name'] . '</h1>'); echo('<p>' . $this->data['available.rules'][$this->data['selected.rule']]['descr'] . '</p>'); - +// echo('<pre>'); +// print_r($this->data); +// exit; // Report settings @@ -73,7 +102,9 @@ echo('<tr><td style="width: 50px; padding: 0px"><img style="margin: 0px" src="' // Select report echo '<td>'; -echo '<form style="display: inline"><select onChange="submit();" name="rule">'; +echo '<form style="display: inline">'; +echo getBaseURL($this, 'post', 'rule'); +echo '<select onChange="submit();" name="rule">'; foreach ($this->data['available.rules'] AS $key => $rule) { if ($key === $this->data['selected.rule']) { echo '<option selected="selected" value="' . $key . '">' . $rule['name'] . '</option>'; @@ -91,8 +122,7 @@ echo '<td style="text-align: right">'; #echo('<pre>here'); print_r($this->data['delimiterPresentation']); echo('</pre>'); echo '<form style="display: inline">'; -echo '<input type="hidden" name="rule" value="' . $this->data['selected.rule'] . '" />'; -echo '<input type="hidden" name="time" value="' . $this->data['selected.time'] . '" />'; +echo getBaseURL($this, 'post', 'd'); echo '<select onChange="submit();" name="d">'; foreach ($this->data['availdelimiters'] AS $key => $delim) { @@ -121,15 +151,39 @@ echo '</table>'; echo '<table class="selecttime" style="width: 100%; border: 1px solid #ccc; background: #eee; margin: 1px 0px; padding: 0px">'; echo('<tr><td style="width: 50px; padding: 0px"><img style="margin: 0px" src="' . SimpleSAML_Module::getModuleURL("statistics/resources/calendar.png") . '" alt="Select date and time" /></td>'); + + + + + if (isset($this->data['available.times.prev'])) { - echo('<td style=""><a href="showstats.php?rule=' . $this->data['selected.rule']. '&time=' . $this->data['available.times.prev'] . '">« Previous</a></td>'); + + echo('<td style=""><a href="' . getBaseURL($this, 'get', 'time', $this->data['available.times.prev']) . '">« Previous</a></td>'); } else { echo('<td style="color: #ccc">« Previous</td>'); } -echo '<td style="text-align: center">'; + +echo '<td style="text-align: right">'; +echo '<form style="display: inline">'; +echo getBaseURL($this, 'post', 'res'); +// echo '<input type="hidden" name="rule" value="' . $this->data['selected.rule'] . '" />'; +echo '<select onChange="submit();" name="res">'; +foreach ($this->data['available.timeres'] AS $key => $timeresname) { + if ($key == $this->data['selected.timeres']) { + echo '<option selected="selected" value="' . $key . '">' . $timeresname . '</option>'; + } else { + echo '<option value="' . $key . '">' . $timeresname . '</option>'; + } +} +echo '</select></form>'; +echo '</td>'; + + +echo '<td style="text-align: left">'; echo '<form style="display: inline">'; -echo '<input type="hidden" name="rule" value="' . $this->data['selected.rule'] . '" />'; +echo getBaseURL($this, 'post', 'time'); +// echo '<input type="hidden" name="rule" value="' . $this->data['selected.rule'] . '" />'; echo '<select onChange="submit();" name="time">'; foreach ($this->data['available.times'] AS $key => $timedescr) { if ($key == $this->data['selected.time']) { @@ -142,7 +196,7 @@ echo '</select></form>'; echo '</td>'; if (isset($this->data['available.times.next'])) { - echo('<td style="text-align: right; padding-right: 4px"><a href="showstats.php?rule=' . $this->data['selected.rule']. '&time=' . $this->data['available.times.next'] . '">Next »</a></td>'); + echo('<td style="text-align: right; padding-right: 4px"><a href="' . getBaseURL($this, 'get', 'time', $this->data['available.times.next']) . '">Next »</a></td>'); } else { echo('<td style="color: #ccc; text-align: right; padding-right: 4px">Next »</td>'); } diff --git a/modules/statistics/templates/statmeta-tpl.php b/modules/statistics/templates/statmeta-tpl.php new file mode 100644 index 0000000000000000000000000000000000000000..1ad37d3c559b27a9410f0add0571ac9298060bc9 --- /dev/null +++ b/modules/statistics/templates/statmeta-tpl.php @@ -0,0 +1,56 @@ +<?php +$this->data['header'] = 'SimpleSAMLphp Statistics Metadata'; +$this->includeAtTemplateBase('includes/header.php'); + +?> + + +<?php + +echo('<table style="width: 100%">'); + +if (isset($this->data['metadata'])) { + + if (isset($this->data['metadata']['lastrun'] )) { + echo('<tr><td>Aggregator last run at</td><td>' . + date('l jS \of F Y H:i:s', $this->data['metadata']['lastrun']) . + '</td></tr>'); + } + + if (isset($this->data['metadata']['notBefore'] )) { + echo('<tr><td>Aggregated data until</td><td>' . + date('l jS \of F Y H:i:s', $this->data['metadata']['notBefore']) . + '</td></tr>'); + } + + if (isset($this->data['metadata']['memory'] )) { + echo('<tr><td>Memory usage</td><td>' . + number_format($this->data['metadata']['memory'] / (1024*1024), 2) . ' MB' . + '</td></tr>'); + } + if (isset($this->data['metadata']['time'] )) { + echo('<tr><td>Execution time</td><td>' . + $this->data['metadata']['time'] . ' seconds' . + '</td></tr>'); + } + if (isset($this->data['metadata']['lastlinehash'] )) { + echo('<tr><td>SHA1 of last processed logline</td><td>' . + $this->data['metadata']['lastlinehash'] . + '</td></tr>'); + } + if (isset($this->data['metadata']['lastline'] )) { + echo('<tr><td>Last processed logline</td><td>' . + $this->data['metadata']['lastline'] . + '</td></tr>'); + } + + +} else { + echo('<tr><td>No metadata found</td></tr>'); +} + +echo('</table>'); + +echo('<p>[ <a href="showstats.php">Show statistics</a> ] </p>'); + +$this->includeAtTemplateBase('includes/footer.php'); diff --git a/modules/statistics/www/showstats.php b/modules/statistics/www/showstats.php index bc790862e54e63210368b227f9d6d8a444e17b73..20ef8c4c83fbab8496e5e62e2840804ff61101bb 100644 --- a/modules/statistics/www/showstats.php +++ b/modules/statistics/www/showstats.php @@ -5,6 +5,9 @@ $statconfig = SimpleSAML_Configuration::getConfig('module_statistics.php'); $session = SimpleSAML_Session::getInstance(); +/** + * AUTHENTICATION and Authorization for access to the statistics. + */ $protected = $statconfig->getBoolean('protected', FALSE); $authsource = $statconfig->getString('auth', NULL); $allowedusers = $statconfig->getValue('allowedUsers', NULL); @@ -44,320 +47,93 @@ if ($protected) { SimpleSAML_Utilities::requireAdmin(); } } - - - - -$statdir = $statconfig->getValue('statdir'); -$inputfile = $statconfig->getValue('inputfile'); -$statrules = $statconfig->getValue('statrules'); - -$datehandler = new sspmod_statistics_DateHandler($statconfig->getValue('offset', 0)); - - - -/* - * Walk through file lists, and get available [rule][fileslot]... - */ -if (!is_dir($statdir)) - throw new Exception('Statisics output directory [' . $statdir . '] does not exists.'); -$filelist = scandir($statdir); -$available = array(); -foreach ($filelist AS $file) { - if (preg_match('/([a-z0-9_]+)-([0-9]+)\.stat/', $file, $matches)) { - if (array_key_exists($matches[1], $statrules)) { - $available[$matches[1]][] = $matches[2]; - } - } -} -if (empty($available)) - throw new Exception('No aggregated statistics files found in [' . $statdir . ']'); - -/* - * Create array with information about available rules.. - */ -$available_rules = array(); -foreach ($available AS $key => $av) { - $available_rules[$key] = array('name' => $statrules[$key]['name'], 'descr' => $statrules[$key]['descr']); -} -$availrulenames = array_keys($available_rules); - -// Get selected rulename.... -$rule = $statconfig->getString('default', $availrulenames[0]); -if(array_key_exists('rule', $_GET)) { - if (array_key_exists($_GET['rule'], $available_rules)) { - $rule = $_GET['rule']; - } -} - - - - - -/* - * Get list of avaiable times in current file (rule) +/** + * AUTHENTICATION and Authorization for access to the statistics. ------ */ -$available_times = array(); -foreach ($available[$rule] AS $slot) { - $available_times[$slot] = $datehandler->prettyDateSlot($slot, $statrules[$rule]['fileslot'], $statrules[$rule]['dateformat-period']) . - ' to ' . $datehandler->prettyDateSlot($slot+1, $statrules[$rule]['fileslot'], $statrules[$rule]['dateformat-period']); -} - -// Get which time (fileslot) to use.. First get a default, which is the most recent one. -$fileslot = $available[$rule][count($available[$rule])-1]; -// Then check if the user have provided one. -if (array_key_exists('time', $_GET)) { - if (in_array($_GET['time'], $available[$rule])) { - $fileslot = $_GET['time']; - } -} - -// Extract previous and next time slots... -$available_times_prev = NULL; $available_times_next = NULL; -$timeslots = array_keys($available_times); -$timeslotindex = array_flip($timeslots); - -if ($timeslotindex[$fileslot] > 0) - $available_times_prev = $timeslots[$timeslotindex[$fileslot]-1]; -if ($timeslotindex[$fileslot] < (count($timeslotindex)-1) ) - $available_times_next = $timeslots[$timeslotindex[$fileslot]+1]; - - - -// Get file and extract results. -$resultFileName = $statdir . $rule . '-' . $fileslot . '.stat'; -if (!file_exists($resultFileName)) - throw new Exception('Aggregated statitics file [' . $resultFileName . '] not found.'); -if (!is_readable($resultFileName)) - throw new Exception('Could not read statitics file [' . $resultFileName . ']. Bad file permissions?'); -$resultfile = file_get_contents($resultFileName); -$results = unserialize($resultfile); -if (empty($results)) - throw new Exception('Aggregated statistics in file [' . $resultFileName . '] was empty.'); /* - * Get rule specific configuration from the configuration file. + * Check input parameters */ -$slotsize = $statrules[$rule]['slot']; -$dateformat_period = $statrules[$rule]['dateformat-period']; -$dateformat_intra = $statrules[$rule]['dateformat-intra']; -$axislabelint = $statrules[$rule]['axislabelint']; +$preferRule = NULL; +$preferTime = NULL; +$preferTimeRes = NULL; +$delimiter = NULL; +if(array_key_exists('rule', $_GET)) $preferRule = $_GET['rule']; +if(array_key_exists('time', $_GET)) $preferTime = $_GET['time']; +if(array_key_exists('res', $_GET)) $preferTimeRes = $_GET['res']; +if(array_key_exists('d', $_GET)) $delimiter = $_GET['d']; -$delimiter = '_'; -if (isset($_REQUEST['d'])) { - $delimiter = $_REQUEST['d']; -} /* - * Walk through dataset to get the max values. + * Create statistics data. */ -$maxvalue = 0; -$maxvaluetime = 0; -$debugdata = array(); +$ruleset = new sspmod_statistics_Ruleset($statconfig); +$statrule = $ruleset->getRule($preferRule); +$rule = $statrule->getRuleID(); +$dataset = $statrule->getDataset($preferTimeRes, $preferTime); +$dataset->setDelimiter($delimiter); +$delimiter = $dataset->getDelimiter(); -/* - * Search for maximum value in order to scale the Y-scale of the graph presentation. - */ -$maxdelimiter = $delimiter; -if (array_key_exists('graph.total', $statrules[$rule]) && $statrules[$rule]['graph.total'] === TRUE) { - $maxdelimiter = '_'; -} - -foreach($results AS $slot => &$res) { - if (!array_key_exists($maxdelimiter, $res)) $res[$maxdelimiter] = 0; - if ($res[$maxdelimiter] > $maxvalue) { - $maxvaluetime = $datehandler->prettyDateSlot($slot, $slotsize, $dateformat_intra); - } - $maxvalue = max($res[$maxdelimiter],$maxvalue); - $debugdata[$slot] = array($datehandler->prettyDateSlot($slot, $slotsize, $dateformat_intra), $res[$maxdelimiter] ); -} -$max = sspmod_statistics_Graph_GoogleCharts::roof($maxvalue); - -#echo 'Maxvalue [' . $maxvalue . '] at time ' . $maxvaluetime; exit; -#echo '<pre>'; print_r($debugdata); exit; +$timeres = $dataset->getTimeRes(); +$fileslot = $dataset->getFileslot(); +$availableFileSlots = $statrule->availableFileSlots($timeres); +$timeNavigation = $statrule->getTimeNavigation($timeres, $preferTime); +$dataset->aggregateSummary(); +$dataset->calculateMax(); -/** - * Aggregate summary table from dataset. To be used in the table view. - */ -$summaryDataset = array(); -foreach($results AS $slot => $res) { - foreach ($res AS $key => $value) { - if (array_key_exists($key, $summaryDataset)) { - $summaryDataset[$key] += $value; - } else { - $summaryDataset[$key] = $value; - } - } -} -asort($summaryDataset); -$summaryDataset = array_reverse($summaryDataset, TRUE); -// echo '<pre>'; print_r($summaryDataset); exit; - -/* - * Create a list of delimiter keys that has the highest total summary in this period. - */ -$topdelimiters = array(); -$maxdelimiters = 4; $i = 0; -foreach($summaryDataset AS $key => $value) { - if ($key !== '_') - $topdelimiters[] = $key; - if ($i++ >= $maxdelimiters) break; -} - -// echo('<pre>'); print_r($topdelimiters); exit; - -$piedata = array(); $sum = 0; -foreach($topdelimiters AS $td) { - $sum += $summaryDataset[$td]; - $piedata[] = number_format(100*$summaryDataset[$td] / $summaryDataset['_'], 2); -} -$piedata[] = number_format(100 - 100*($sum /$summaryDataset['_']), 2); +$piedata = $dataset->getPieData(); $datasets = array(); +$datasets[] = $dataset->getPercentValues(); -#$max = 25; -$availdelimiters = array(); -$xentries = count($results); -$lastslot = 0; $i = 0; - - -/* - * Walk through dataset to get percent values from max into dataset[]. - */ -function getPercentValues($results, $delimiter) { - - #echo('<pre>'); print_r($results); exit; - - global $slot, $slotsize, $dateformat_intra, $axis, $lastslot, $axispos, $availdelimiters, $max, $datehandler, $axislabelint, $i, $xentries; - - $axis = array(); - $axispos = array(); - - $dataset = array(); - foreach($results AS $slot => $res) { - #echo ('<p>new value: ' . number_format(100*$res[$delimiter] / $max, 2)); -// echo('<hr><p>delimiter [<tt>' .$delimiter . '</tt>].'); -// echo('<p>Res <pre>'); print_r($res); echo( '</pre>'); -// echo('<p>return <pre>'); print_r(isset($res[$delimiter]) ? $res[$delimiter] : 'NO'); echo('</pre>'); - if (array_key_exists($delimiter, $res)) { - if ($res[$delimiter] === NULL) { - $dataset[] = -1; - } else { - $dataset[] = number_format(100*$res[$delimiter] / $max, 2); - } - } else { - $dataset[] = '0'; - } - foreach(array_keys($res) AS $nd) $availdelimiters[$nd] = 1; - - // check if there should be an axis here... - if ( $slot % $axislabelint == 0) { - $axis[] = $datehandler->prettyDateSlot($slot, $slotsize, $dateformat_intra); - $axispos[] = (($i)/($xentries-1)); - // echo 'set axis on [' . $slot . '] = [' . $datehandler->prettyDateSlot($slot, $slotsize, $dateformat_intra) . ']'; - } - $lastslot = $slot; - $i++; - } - - return $dataset; -} - - - - -$datasets[] = getPercentValues($results, $delimiter); -if ($delimiter !== '_') { - if (array_key_exists('graph.total', $statrules[$rule]) && $statrules[$rule]['graph.total'] === TRUE) { - $datasets[] = getPercentValues($results, '_'); - } -} - - - - - - - - - - - -#echo('<pre>'); print_r($datasets); exit; - -#echo 'set axis on lastslot [' . $lastslot . ']'; -$axis[] = $datehandler->prettyDateSlot($lastslot+1, $slotsize, $dateformat_intra); -#print_r($axis); - -#echo('<pre>'); print_r($axis); exit; +$axis = $dataset->getAxis(); +$max = $dataset->getMax(); $dimx = $statconfig->getValue('dimension.x', 800); $dimy = $statconfig->getValue('dimension.y', 350); $grapher = new sspmod_statistics_Graph_GoogleCharts($dimx, $dimy); - $htmlContentPre = array(); $htmlContentPost = array(); $htmlContentHead = array(); $jquery = array(); $hookinfo = array('pre' => &$htmlContentPre, 'post' => &$htmlContentPost, 'head' => &$htmlContentHead, 'jquery' => &$jquery, 'page' => 'statistics'); SimpleSAML_Module::callHooks('htmlinject', $hookinfo); $t = new SimpleSAML_XHTML_Template($config, 'statistics:statistics-tpl.php'); - - -/* - * Create a delimiter presentation filter for this rule... - */ -$delimiterPresentation = NULL; -if (array_key_exists('fieldPresentation', $statrules[$rule])) { - $classname = SimpleSAML_Module::resolveClass( $statrules[$rule]['fieldPresentation']['class'], 'Statistics_FieldPresentation'); - if (!class_exists($classname)) - throw new Exception('Could not find field presentation plugin [' . $classname . ']: No class found'); - - $presentationHandler = new $classname(array_keys($availdelimiters), $statrules[$rule]['fieldPresentation']['config'], $t); - $delimiterPresentation = $presentationHandler->getPresentation(); -} - - -$pieaxis = array(); -foreach($topdelimiters AS $key) { - $keyName = $key; - if(array_key_exists($key, $delimiterPresentation)) $keyName = $delimiterPresentation[$key]; - $pieaxis[] = $keyName; -} -$pieaxis[] = 'Others'; - - - $t->data['header'] = 'stat'; -$t->data['imgurl'] = $grapher->show($axis, $axispos, $datasets, $max); - -$t->data['pieimgurl'] = $grapher->showPie($pieaxis, $piedata); - -$t->data['available.rules'] = $available_rules; -$t->data['available.times'] = $available_times; -$t->data['available.times.prev'] = $available_times_prev; -$t->data['available.times.next'] = $available_times_next; +$t->data['imgurl'] = $grapher->show($axis['axis'], $axis['axispos'], $datasets, $max); +$t->data['pieimgurl'] = $grapher->showPie( $dataset->getDelimiterPresentationPie(), $piedata); +$t->data['available.rules'] = $ruleset->availableRulesNames(); +$t->data['available.times'] = $statrule->availableFileSlots($timeres); +$t->data['available.timeres'] = $statrule->availableTimeRes(); +$t->data['available.times.prev'] = $timeNavigation['prev']; +$t->data['available.times.next'] = $timeNavigation['next']; $t->data['htmlContentPre'] = $htmlContentPre; $t->data['htmlContentPost'] = $htmlContentPost; $t->data['htmlContentHead'] = $htmlContentHead; $t->data['jquery'] = $jquery; $t->data['selected.rule']= $rule; $t->data['selected.time'] = $fileslot; -$t->data['debugdata'] = $debugdata; -$t->data['results'] = $results; -$t->data['summaryDataset'] = $summaryDataset; -$t->data['topdelimiters'] = $topdelimiters; -$t->data['availdelimiters'] = array_keys($availdelimiters); -$t->data['delimiterPresentation'] = $delimiterPresentation; +$t->data['selected.timeres'] = $timeres; +$t->data['selected.delimiter'] = $delimiter; + +$t->data['debugdata'] = $dataset->getDebugData(); +$t->data['results'] = $dataset->getResults(); +$t->data['summaryDataset'] = $dataset->getSummary(); +$t->data['topdelimiters'] = $dataset->getTopDelimiters(); +$t->data['availdelimiters'] = $dataset->availDelimiters(); + + + +$t->data['delimiterPresentation'] = $dataset->getDelimiterPresentation(); $t->show(); diff --git a/modules/statistics/www/statmeta.php b/modules/statistics/www/statmeta.php new file mode 100644 index 0000000000000000000000000000000000000000..8b99a686f6f461656ab6de297e6c9a0787bf5283 --- /dev/null +++ b/modules/statistics/www/statmeta.php @@ -0,0 +1,64 @@ +<?php + +$config = SimpleSAML_Configuration::getInstance(); +$statconfig = SimpleSAML_Configuration::getConfig('module_statistics.php'); +$session = SimpleSAML_Session::getInstance(); + + +/** + * AUTHENTICATION and Authorization for access to the statistics. + */ +$protected = $statconfig->getBoolean('protected', FALSE); +$authsource = $statconfig->getString('auth', NULL); +$allowedusers = $statconfig->getValue('allowedUsers', NULL); +$useridattr = $statconfig->getString('useridattr', 'eduPersonPrincipalName'); + +if ($protected) { + + if (SimpleSAML_Utilities::isAdmin()) { + // User logged in as admin. OK. + SimpleSAML_Logger::debug('Statistics auth - logged in as admin, access granted'); + + } elseif(isset($authsource) && $session->isValid($authsource) ) { + + // User logged in with auth source. + SimpleSAML_Logger::debug('Statistics auth - valid login with auth source [' . $authsource . ']'); + + // Retrieving attributes + $attributes = $session->getAttributes(); + + // Check if userid exists + if (!isset($attributes[$useridattr])) + throw new Exception('User ID is missing'); + + // Check if userid is allowed access.. + if (!in_array($attributes[$useridattr][0], $allowedusers)) { + SimpleSAML_Logger::debug('Statistics auth - User denied access by user ID [' . $attributes[$useridattr][0] . ']'); + throw new Exception('Access denied for this user.'); + } + SimpleSAML_Logger::debug('Statistics auth - User granted access by user ID [' . $attributes[$useridattr][0] . ']'); + + } elseif(isset($authsource)) { + // If user is not logged in init login with authrouce if authsousrce is defined. + SimpleSAML_Auth_Default::initLogin($authsource, SimpleSAML_Utilities::selfURL()); + + } else { + // If authsource is not defined, init admin login. + SimpleSAML_Utilities::requireAdmin(); + } +} + +$aggr = new sspmod_statistics_Aggregator(); +$aggr->loadMetadata(); +$metadata = $aggr->getMetadata(); + +// echo('<pre>'); print_r($metadata); + +/** + * AUTHENTICATION and Authorization for access to the statistics. ------ + */ + +$t = new SimpleSAML_XHTML_Template($config, 'statistics:statmeta-tpl.php'); +$t->data['metadata'] = $metadata; +$t->show(); +