Impemented logpeek to do blockwise reading of logfile for faster execution.

Patch by Thomas Graff <thomas.graff@uninett.no>

git-svn-id: https://simplesamlphp.googlecode.com/svn/trunk@1474 44740490-163a-0410-bde0-09ae8108e29a

modules/logpeek/config-templates/module_logpeek.php

@@ -8,6 +8,8 @@
 $config = array (
 	'logfile'	=> '/var/log/simplesamlphp.log',
 	'lines'		=> 1500,
+	// Read block size. 8192 is max, limited by fread.
+	'blocksz'	=> 8192,
 );
 
 ?>

modules/logpeek/lib/File/reverseRead.php

+<?php
+/**
+ * Functionatility for line by line reverse reading of a file. It is done by blockwise
+ * fetching the file from the end and putting the lines into an array.
+ * 
+ * @author Thomas Graff<thomas.graff@uninett.no>
+ *
+ */
+class sspmod_logpeek_File_reverseRead{
+	// 8192 is max number of octets limited by fread.
+	private $blockSize;
+	private $blockStart;
+	private $fileHandle;
+	// fileSize may be changed after initial file size check
+	private $fileSize;
+	private $fileMtime;
+	// Array containing file lines
+	private $content;
+	// Leftover before first complete line
+	private $remainder;
+	// Count read lines from the end
+	private $readPointer;
+	
+	/**
+	 * File is checked and file handle to file is opend. But no data is read
+	 * from the file.
+	 * 
+	 * @param string $fileUrl Path and filename to file to be read
+	 * @param int $blockSize File read block size in byte
+	 * @return bool Success
+	 */
+	public function __construct($fileUrl, $blockSize = 8192){
+		if(!is_readable($fileUrl)){
+			return FALSE;
+		}
+		
+		$this->blockSize = $blockSize;
+		$this->content = array();
+		$this->remainder = '';
+		$this->readPointer = 0;
+		
+		$fileInfo = stat($fileUrl);
+		$this->fileSize = $this->blockStart = $fileInfo['size'];
+		$this->fileMtime = $fileInfo['mtime'];
+		
+		if($this->fileSize > 0){
+			$this->fileHandle = fopen($fileUrl, 'rb');
+			return TRUE;
+		}else{
+			return FALSE;
+		}
+	}
+	
+	
+	public function __destruct(){
+		if(is_resource($this->fileHandle)){
+			fclose($this->fileHandle);
+		}
+	}
+	
+	/**
+	 * Fetch chunk of data from file.
+	 * Each time this function is called, will it fetch a chunk
+	 * of data from the file. It starts from the end of the file
+	 * and work towards the beginning of the file.
+	 * 
+	 * @return string buffer with datablock.
+	 * Will return bool FALSE when there is no more data to get.
+	 */
+	private function readChunk(){
+		$splits = $this->blockSize;
+		
+		$this->blockStart -= $splits;
+		if($this->blockStart < 0){
+			$splits += $this->blockStart;
+			$this->blockStart = 0;
+		}
+		
+		// Return false if nothing more to read
+		if($splits === 0){
+			return FALSE;
+		}
+		
+		fseek($this->fileHandle, $this->blockStart, SEEK_SET);
+		$buff = fread($this->fileHandle, $splits);
+		
+		// $buff = stream_get_contents($this->fileHandle, $splits, $this->blockStart);
+		
+		return $buff;
+	}
+	
+	/**
+	 * Get one line of data from the file, starting from the end of the file.
+	 * 
+	 * @return string One line of data from the file.
+	 * Bool FALSE when there is no more data to get.
+	 */
+	public function getPreviousLine(){
+		if(count($this->content) === 0 || $this->readPointer < 1){
+			
+			do {
+				$buff = $this->readChunk();
+				
+				if($buff !== FALSE){
+					$eolPos = strpos($buff, "\n");
+				}else{
+					// Empty buffer, no more to read.
+					if(strlen($this->remainder) > 0){
+						$buff = $this->remainder;
+						$this->remainder = '';
+						// Exit from while-loop
+						break;
+					}else{
+						// Remainder also empty.
+						return FALSE;
+					}
+				}
+				
+				if($eolPos === FALSE){
+					// No eol found. Make buffer head of remainder and empty buffer.
+					$this->remainder = $buff . $this->remainder;
+					$buff = '';
+				}elseif($eolPos !== 0){
+					// eol found.
+					$buff .= $this->remainder;
+					$this->remainder = substr($buff, 0, $eolPos);
+					$buff = substr($buff, $eolPos+1);
+				}elseif($eolPos === 0){
+					$buff .= $this->remainder;
+					$buff = substr($buff, 1);
+					$this->remainder = '';
+				}
+				
+			}while(($buff !== FALSE) && ($eolPos === FALSE));
+			
+			$this->content = explode("\n", $buff);
+			$this->readPointer = count($this->content);
+		}
+		
+		if(count($this->content) > 0){
+			return $this->content[--$this->readPointer];
+		}else{
+			return FALSE;
+		}
+	}
+	
+	
+	private function cutHead(&$haystack, $needle, $exit){
+		$pos = 0;
+		$cnt = 0;
+		// Holder på inntill antall ønskede linjer eller vi ikke finner flere linjer
+		while($cnt < $exit && ($pos = strpos($haystack, $needle, $pos)) !==false ){
+			$pos++;
+			$cnt++;
+		}   
+		return $pos == false? false: substr($haystack, $pos, strlen($haystack));
+	}
+	
+	
+	// FIXME: This function hawe som error, do not use before auditing and testing
+	public function getTail($lines = 10){
+		$this->blockStart = $this->fileSize;
+		$buff1 = Array();
+		$lastLines = array();
+		
+		while($this->blockStart){
+			$buff = $this->readChunk();
+			if(!$buff)break;
+			
+			$lines -= substr_count($buff, "\n");
+			
+			if($lines <= 0)
+			{
+				$buff1[] = $this->cutHead($buff, "\n", abs($lines)+1);
+				break;
+			}
+			$buff1[] = $buff;
+		}
+		
+		for($i = count($buff1); $i >= 0; $i--){
+			$lastLines = array_merge($lastLines, explode("\n", $buff1[$i]));
+		}
+		
+		return $lastLines;
+		// return str_replace("\r", '', implode('', array_reverse($buff1)));
+	}
+	
+	
+	private function getLineAtPost($pos){
+		if($pos < 0 || $pos > $this->fileSize){
+			return FALSE;
+		}
+		
+		$seeker = $pos;
+		fseek($this->fileHandle, $seeker, SEEK_SET);
+		while($seeker > 0 && fgetc($this->fileHandle) !== "\n"){
+			fseek($this->fileHandle, --$seeker, SEEK_SET);
+		}
+		
+		return rtrim(fgets($this->fileHandle));
+	}
+	
+	
+	public function getFirstLine(){
+		return $this->getLineAtPost(0);
+	}
+	
+	
+	public function getLastLine(){
+		return $this->getLineAtPost($this->fileSize-2);
+	}
+	
+	
+	public function getFileSize(){
+		return $this->fileSize;
+	}
+	
+	
+	public function getFileMtime(){
+		return $this->fileMtime;
+	}
+	
+}
+?>
\ No newline at end of file

modules/logpeek/lib/Syslog/parseLine.php

+<?php
+class sspmod_logpeek_Syslog_parseLine{
+	
+	
+	public static function isOlderThan($time, $logLine){
+		return true;
+	}
+	
+	public static function getUnixTime($logLine, $year = NULL){
+		// I can read month and day and time from the file.
+		// but I will assum year is current year retured by time().
+		// Unless month and day in the file is bigger than current month and day,
+		// I will then asume prevous year.
+		// A better approach would be to get the year from last modification time (mtime) of the
+		// file this record is taken from. But that require knowledge about the file.
+		if(!$year){
+			$now = getdate();
+			$year = (int)$now['year'];
+		}
+		list($month, $day, $hour, $minute, $second) = sscanf($logLine, "%s %d %d:%d:%d ");
+		$time = sprintf("%d %s %d %d:%d:%d", $day, $month, $year, $hour, $minute, $second);
+		return strtotime($time);
+	}
+}
+?>
\ No newline at end of file

modules/logpeek/templates/logpeek.php

 <?php
 $this->data['header'] = 'Log peek';
 $this->includeAtTemplateBase('includes/header.php');
-
-
 ?>
 
 <h2>SimpleSAMLphp logs (admin utility)</h2>
 
 <form method="get" action="?">
-	<input type="text" name="tag" value="<?php echo $this->data['trackid']; ?>" />
-	<input type="submit" value="Show logs" />
+	<table>
+		<tr><th><label for="start">First entry in logfile</label></th><td id="star"><?php echo $this->data['timestart']; ?></td></tr>
+		<tr><th><label for="end">Last entry in logfile</label></th><td id="end"><?php echo $this->data['endtime']; ?></td></tr>
+		<tr><th><label for="size">Logfile size</label></th><td id="size"><?php echo $this->data['filesize']; ?></td></tr>
+		<tr><th><label for="tag">Tag id for search</label></th><td><input type="text" name="tag" id="tag" value="<?php echo $this->data['trackid']; ?>" /></td></tr>
+		<tr><th><input type="submit" value="Search log" /></th><td></td></tr>
+	</table>
 </form>
 
-
 <pre style="background: #eee; border: 1px solid #666; padding: 1em; margin: .4em; overflow: scroll">
 <?php
-
 if (!empty($this->data['results'])) {
 	foreach($this->data['results'] AS $line) {
-		echo $line;
+		echo htmlspecialchars($line) . "\n";
 	}
 }
-
 ?>
 </pre>
 <?php $this->includeAtTemplateBase('includes/footer.php'); ?>
\ No newline at end of file

modules/logpeek/www/index.php

 <?php
 
+function logFilter($objFile, $tag, $cut){
+	if (!preg_match('/^[a-f0-9]{10}$/', $tag)) throw new Exception('Invalid search tag');
+	
+	$i = 0;
+	$results = array();
+	$line = $objFile->getPreviousLine();
+	while($line !== FALSE && ($i++ < $cut)){
+		if(strstr($line, '[' . $tag . ']')){
+			$results[] = $line;
+		}
+		$line = $objFile->getPreviousLine();
+	}
+	$results[] = 'Searched ' . $i . ' lines backward. ' . count($results) . ' lines found.';
+	$results = array_reverse($results);
+	return $results;
+}
+
 
 $config = SimpleSAML_Configuration::getInstance();
 $session = SimpleSAML_Session::getInstance();
 
 SimpleSAML_Utilities::requireAdmin();
 
-
 $logpeekconfig = SimpleSAML_Configuration::getConfig('module_logpeek.php');
-
 $logfile = $logpeekconfig->getValue('logfile', '/var/simplesamlphp.log');
+$blockSize = $logpeekconfig->getValue('blocksz', 8192);
 
-function grepLog($logfile, $tag, $lines) {
+$myLog = new sspmod_logpeek_File_reverseRead($logfile, $blockSize);
 
-	if (!is_readable($logfile)) throw new Exception('Log file [' . $logfile . '] is not readable. Consider checking the file permissions');
-	if (!preg_match('/^[a-f0-9]{10}$/', $tag)) throw new Exception('Invalid search tag');
-	
-	$results = array();
-	$i=0 ;
-	$line = '';
-	$fp = fopen($logfile,"r") ;
-	if(is_resource($fp)){
-		fseek($fp,0,SEEK_END) ;
-		$a = ftell($fp) ;
-		while($i <= $lines){
-			if(fgetc($fp) == "\n"){
-				$line = fgets($fp);
-				$i++ ;
-				if (strstr($line, '[' . $tag . ']')) 
-					$results[] = $line;
-			}
-			fseek($fp,$a);
-			$a-- ;
-		}
-	}
-
-	$results[] = 'Start search line (' . $lines . ' lines back): ' . substr($line,0,40) . '...';
-	$results = array_reverse($results);
-	return $results;
-}
 
 $results = NULL;
 if (isset($_REQUEST['tag'])) {
-	$results = grepLog($logfile, $_REQUEST['tag'], $logpeekconfig->getValue('lines', 500));
-// 	echo('<pre>log:');
-// 	print_r($results);
+	$results = logFilter($myLog, $_REQUEST['tag'], $logpeekconfig->getValue('lines', 500));
 }
 
 
+$fileModYear = date("Y", $myLog->getFileMtime());
+$firstLine = $myLog->getFirstLine();
+$firstTimeEpoch = sspmod_logpeek_Syslog_parseLine::getUnixTime($firstLine, $fileModYear);
+$lastLine = $myLog->getLastLine();
+$lastTimeEpoch = sspmod_logpeek_Syslog_parseLine::getUnixTime($lastLine, $fileModYear);
+$fileSize = $myLog->getFileSize();
 
 $t = new SimpleSAML_XHTML_Template($config, 'logpeek:logpeek.php');
 $t->data['results'] = $results;
 $t->data['trackid'] = $session->getTrackID();
-$t->show();
-exit;
+$t->data['timestart'] = date(DATE_RFC822, $firstTimeEpoch);
+$t->data['endtime'] = date(DATE_RFC822, $lastTimeEpoch);
+$t->data['filesize'] = $fileSize;
 
-?>
+$t->show();
+?>
\ No newline at end of file