Export all pages, simplified datefrom, and suppress fix

README.md

@@ -4,13 +4,19 @@
 Adiscon LogAnalyzer, a web frontend to log data from the same folks the created rsyslog
 
 # todo
- - export: add checkbox to export full filtered history (now exports only current page)
- - export: place ts into export filename (range from-to)
- - BUG: "Suppress duplicated messages" doesn't work
- - filter: allow to specify AFTER:n BEFORE:n (if possible/fast to implement) <- include records before and after match
- - export: configure columns for file export (allow to remove unnecessary columns) <- exclude list of columns
  - BUG: sometimes spinner on index page is drawn in the middle of page irrespective of it's size, but should be drawn in the middle of screen
-
+ - GUI: add checkbox on events page to suppress duplicates in export (override default ExportSuppressDuplicatedMessages
+ - GUI: add checkbox ... to override ExportAllMatchPages
+
+# changes 230122
+ - export: append exported range timestamp to filename
+ - filter: dateto/datefrom add support for offset from current date, i.e. datefrom:T00:00:00 dateto:T01:00:00, datefrom:\-2T, datefrom:\-1T01
+ - fixed BUG: "Suppress duplicated messages" + add property to control distance between duplicates
+ - GUI: if suppress is enabled, then show how much records were suppressed (LN_GEN_SUPPRESSEDRECORDCOUNT)
+ - $CFG[ExportSuppressDuplicatedMessages] - separate export suppressing from view
+ - $CFG[ExportAllMatchPages] - allow to control how much to export by default
+ - $CFG[DuplicateRecordMaxTsDistance] - check duplicates timestamp before suppressing, i.e. same message with 24h period may not be assumed as duplicate
+
 # changes 230121
  - datelastx - keep for backward compatibility (for saved searches); add datelastxx
 
@@ -43,8 +49,10 @@ Adiscon LogAnalyzer, a web frontend to log data from the same folks the created
  - gui: add loglevel style colors and change color for full line; 
  - filter: change datelastx behaviour - use number as hours indicator, i.e. datelastx:3 is 3 hours limit
 
- #obsolete
- - filter: allow to OR msg, i.e. key1 &key2 |key3;
+# obsolete
+ - filter: allow to specify AFTER:n BEFORE:n (if possible/fast to implement) <- include records before and after match
+ - export: configure columns for file export (allow to remove unnecessary columns) <- exclude list of columns
+ - filter: allow to OR msg, i.e. key1 &key2 |key3; (instead use regex)
  - filter: date{from,to} - allow to use today/yesterday + short time, i.e. today 1h same as 1h, yesterday 2h, since/after/before/etc.
  - "Maximize view" - reloads page and resets search filter, hide toolbars with js instead
  - changing "Autoreload" does the same as "Max. view"

src/export.php

@@ -75,6 +75,8 @@
 $content['searchstr'] = "";
 $content['error_occured'] = false;
 
+$content['ExportAllMatchPages'] 		= GetConfigSetting("ExportAllMatchPages", 0, CFGLEVEL_USER) == 1;
+$content['SuppressDuplicatedMessages'] 	= GetConfigSetting("SuppressDuplicatedMessages", 0, CFGLEVEL_USER) == 1;
 // Check required input parameters
 if (
 		(isset($_GET['op']) && $_GET['op'] == "export") && 
@@ -203,32 +205,65 @@
 			if ( $ret == SUCCESS )
 			{
 				//Loop through the messages!
+				$duplicateCount = 0; //FIXME while readNext record doesnt properly return skip_status keep it outside of loop body
+				$szLastMessageTimestamp = 0;
+				$DuplicateRecordMaxTsDistance = GetConfigSetting("DuplicateRecordMaxTsDistance", PHP_INT_MAX, CFGLEVEL_USER);
 				do
 				{
 					// --- Extra stuff for suppressing messages
 					if (
-							GetConfigSetting("SuppressDuplicatedMessages", 0, CFGLEVEL_USER) == 1 
-							&&
-							isset($logArray[SYSLOG_MESSAGE])
+						$content['SuppressDuplicatedMessages'] && isset($logArray[SYSLOG_MESSAGE])
 						)
 					{
-
-						if ( !isset($szLastMessage) ) // Only set lastmgr
-							$szLastMessage = $logArray[SYSLOG_MESSAGE];
-						else
+						// Skip if same msg
+						// but don't merge same messages if timestamp difference is greater than precofigured (useful when filtering same messages)
+						$tsDiff = abs($szLastMessageTimestamp - $logArray["timereported"][EVTIME_TIMESTAMP]);
+						//echo "uID=$uID; duplicates ($duplicateCount); delta ts = $tsDiff; isDublicate=".($szLastMessage == $logArray[SYSLOG_MESSAGE])."<br>\n";
+						if ( $szLastMessage == $logArray[SYSLOG_MESSAGE] && ($tsDiff < $DuplicateRecordMaxTsDistance))
 						{
-							// Skip if same msg
-							if ( $szLastMessage == $logArray[SYSLOG_MESSAGE] )
+							$szLastMessageTimestamp = $logArray["timereported"][EVTIME_TIMESTAMP];
+
+							// --- Extra Loop to get the next entry!
+							// FIXME 230122 right now ReadNext skips entries only from custom  msgparser (see: classes/logstreamdb.class.php:601)
+							do
 							{
-								// Set last mgr
-								$szLastMessage = $logArray[SYSLOG_MESSAGE];
+								$ret = $stream->ReadNext($uID, $logArray);
+								$duplicateCount++;
+							} while ( $ret == ERROR_MSG_SKIPMESSAGE );
+							// --- 
+
+							// Skip entry
+							continue;
+						}else{
+							 //inject entry about suppressed records
+							// FIXME any better way of doing that?
+							if($duplicateCount > 1){ //ignore if only 1 duplicate
+								foreach($content['Columns'] as $mycolkey)
+								{
+									if ( isset($fields[$mycolkey]) )
+									{
+										$content['syslogmessages'][$counter][$mycolkey]['FieldColumn'] = $mycolkey;
+										$content['syslogmessages'][$counter][$mycolkey]['uid'] = $uID;
 
-								// Skip entry
-								continue;
+										if($content['fields'][$mycolkey]['FieldType'] == FILTER_TYPE_STRING && $mycolkey == SYSLOG_MESSAGE){
+												$content['syslogmessages'][$counter][$mycolkey]['fieldvalue'] = "... suppressed $duplicateCount duplicate(s)...";
+										}else   $content['syslogmessages'][$counter][$mycolkey]['fieldvalue'] = "\t";
+									}
+								}
+								$counter++;
+					        	$duplicateCount = 0; //reset suppress counter
 							}
+							$szLastMessage = $logArray[SYSLOG_MESSAGE];
+							$szLastMessageTimestamp = $logArray["timereported"][EVTIME_TIMESTAMP];
 						}
 					}
 					// --- 
+
+					if(!isset($content['period_start_ts'])){ //store first record ts
+						$content['period_start_ts'] = $logArray["timereported"][EVTIME_TIMESTAMP];
+					}//*else if(!isset($content['period_start_ts'])){ // FIXME store last record ts
+						$content['period_end_ts'] = $logArray["timereported"][EVTIME_TIMESTAMP];
+					//}
 
 					// --- Now we populate the values array!
 					foreach($content['Columns'] as $mycolkey)
@@ -286,7 +321,17 @@
 
 					// Increment Counter
 					$counter++;
-				} while ($counter < $content['CurrentViewEntriesPerPage'] && ($ret = $stream->ReadNext($uID, $logArray)) == SUCCESS);
+
+					// --- Extra Loop to get the next entry!
+					do
+					{
+						$ret = $stream->ReadNext($uID, $logArray);
+					} while ( $ret == ERROR_MSG_SKIPMESSAGE );
+
+					if(!$content['ExportAllMatchPages'] && $counter >= $content['CurrentViewEntriesPerPage']){
+						break;
+					}
+				} while ($ret == SUCCESS);
 
 				if ( $content['read_direction'] == EnumReadDirection::Forward )
 				{
@@ -331,7 +376,7 @@
 	$szOutputMimeType = "text/plain";
 	$szOutputCharset = "";
 
-	$szOutputFileName = "ExportMessages";
+	$szOutputFileName = "ExportMessages_".date('Ymd\THis',$content['period_start_ts'])."-".date('Ymd\THis',$content['period_end_ts']);
 	$szOutputFileExtension = ".txt";
 	$szOPFieldSeparator = " ";
 	$szOPFirstLineFieldNames = true;

src/include/config.sample.php

@@ -82,6 +82,8 @@
 
 // --- Default Export options
 $CFG['ExportUseTodayYesterday'] = 0;                    // Same as ViewUseTodayYesterday. By default export normal dates
+$CFG['ExportSuppressDuplicatedMessages'] = 0            // If enabled, then export will contain no duplicates (@see related DuplicateRecordMaxTsDistance)
+$CFG['ExportAllMatchPages'] = 0                         // By default export only selected page results
 
 // --- Default Frontend Options 
 $CFG['PrependTitle'] = "";					// If set, this	text will be prepended withint the title tag
@@ -104,6 +106,8 @@
 $CFG['EnableContextLinks'] = 1;				// if enabled, context links within the messages will automatically be created and added. Set this to 0 to disable all context links. 
 $CFG['EnableIPAddressResolve'] = 1;			// If enabled, IP Addresses inline messages are automatically resolved and the result is added in brackets {} behind the IP Address
 $CFG['SuppressDuplicatedMessages'] = 0;		// If enabled, duplicated messages will be suppressed in the main display. 
+$CFG['DuplicateRecordMaxTsDistance'] = PHP_INT_MAX;      // Max timestamp delta between two matching records. If delta is less than this value then records will be suppressed
+
 $CFG['TreatNotFoundFiltersAsTrue'] = 0;		// If you filter / search for messages, and the fields you are filtering for is not found, the filter result is treaten as TRUE! 
 $CFG['PopupMenuTimeout'] = 3000;			// This variable defines the default timeout value for popup menus in milliseconds. (those menus which popup when you click on the value of a field.
 $CFG['PhplogconLogoUrl'] = "";				// Put an Url to a custom toplogo you want to use.

src/include/functions_common.php

@@ -61,7 +61,7 @@
 
 // Try to enable a little more memory in case we do some more filtering
 @ini_set('memory_limit', '512M');
-// ---
+// --- 
 
 // Default language
 $LANG_EN = "en";	// Used for fallback
@@ -1317,6 +1317,9 @@ function RedirectResult( $szMsg, $newpage )
 */
 function GetEventTime($szTimStr)
 {
+	//remove field extra escaping
+	$szTimStr = str_replace("\\","",$szTimStr);
+
 	// Sample: Mar 10 14:45:44
 	if ( preg_match("/(...) ([0-9]{1,2}) ([0-9]{1,2}):([0-9]{1,2}):([0-9]{1,2})/", $szTimStr, $out ) )
 	{
@@ -1393,6 +1396,23 @@ function GetEventTime($szTimStr)
 		$eventtime[EVTIME_TIMEZONE] = date('O'); // Get default Offset
 		$eventtime[EVTIME_MICROSECONDS] = 0;
 	}
+	// Sample: \-1T11:10:50 or \-1T0:1 or T12:0, etc
+	else if ( preg_match("/(-?[0-9]{1,2})?T([0-9]{0,2}):?([0-9]{0,2}):?([0-9]{0,2})/", $szTimStr, $out ) )
+	{
+		$hh = is_numeric($out[2]) ? $out[2] : 0;
+		$mm = is_numeric($out[3]) ? $out[3] : 0;
+		$ss = is_numeric($out[4]) ? $out[4] : 0;
+
+		// RFC 3164 typical timestamp
+		$szTime = mktime($hh, $mm, $ss);
+		if(is_numeric($out[1]) && $out[1] < 0){//day offset
+			$szTime = strtotime($out[1].' days', $szTime);
+		}
+
+		$eventtime[EVTIME_TIMESTAMP] = $szTime;
+		$eventtime[EVTIME_TIMEZONE] = date('O'); // Get default Offset
+		$eventtime[EVTIME_MICROSECONDS] = 0;
+        }
 	else
 	{
 		$eventtime[EVTIME_TIMESTAMP] = 0;
@@ -1733,8 +1753,8 @@ function StartPHPSession()
 	global $RUNMODE;
 	if ( $RUNMODE == RUNMODE_WEBSERVER )
 	{
-		// This will start the session
-		@session_start();
+                // This will start the session
+                @session_start();
 
 
 		if ( !isset($_SESSION['SESSION_STARTED']) )

src/include/functions_filters.php

@@ -323,6 +323,9 @@ function GetMessageTypeDisplayName( $nMsgTypeID )
 
 function GetTimeStampFromTimeString($szTimeString)
 {
+	//remove field extra escaping
+	$szTimeString = str_replace('\\', '', $szTimeString);
+
 	//Sample: 2008-4-1T00:00:00
 	if ( preg_match("/([0-9]{4,4})-([0-9]{1,2})-([0-9]{1,2})T([0-9]{1,2}):([0-9]{1,2}):([0-9]{1,2})$/", $szTimeString, $out) )
 	{
@@ -335,6 +338,20 @@ function GetTimeStampFromTimeString($szTimeString)
 		// return new timestamp
 		return mktime(0,0,0, $out[2], $out[3], $out[1]);
 	}
+	// Sample: \-1T11:10:50 or \-1T0:1 or T12:0, etc
+	else if ( preg_match("/(-?[0-9]{1,2})?T([0-9]{0,2}):?([0-9]{0,2}):?([0-9]{0,2})/", $szTimeString, $out ) )
+	{
+		$hh = is_numeric($out[2]) ? $out[2] : 0;
+		$mm = is_numeric($out[3]) ? $out[3] : 0;
+		$ss = is_numeric($out[4]) ? $out[4] : 0;
+
+		$szTime = mktime($hh, $mm, $ss);
+		if(is_numeric($out[1]) && $out[1] < 0){//day offset
+			$szTime = strtotime($out[1].' days', $szTime);
+		}
+
+		return $szTime;
+    }
 	else
 	{
 		OutputDebugMessage("Unparseable Time in GetTimeStampFromTimeString - '" . $szTimeString . "'", DEBUG_WARN);

src/index.php

@@ -105,6 +105,9 @@
 	$content['skipone'] = false;
 // ---
 
+// Init show suppressed count flag
+$content['SUPPRESS_ENABLED'] = GetConfigSetting("SuppressDuplicatedMessages", 0, CFGLEVEL_USER) == 1;
+
 // Init Export Stuff!
 $content['EXPORT_ENABLED'] = true;
 
@@ -336,38 +339,78 @@
 			// ---
 
 			//Loop through the messages!
+			$duplicateCountTotal = 0;
+			$duplicateCount = 0; //FIXME while readNext record doesnt properly return skip_status keep it outside of loop body
+			$szLastMessageTimestamp = 0;
+			$DuplicateRecordMaxTsDistance = GetConfigSetting("DuplicateRecordMaxTsDistance", PHP_INT_MAX, CFGLEVEL_USER); 
 			do
 			{
 				// --- Extra stuff for suppressing messages
 				if (
-						GetConfigSetting("SuppressDuplicatedMessages", 0, CFGLEVEL_USER) == 1 
-						&&
-						isset($logArray[SYSLOG_MESSAGE])
-					)
+					GetConfigSetting("SuppressDuplicatedMessages", 0, CFGLEVEL_USER) == 1 
+					&&
+					isset($logArray[SYSLOG_MESSAGE])
+				   )
 				{
-
-					if ( !isset($szLastMessage) ) // Only set lastmgr
-						$szLastMessage = $logArray[SYSLOG_MESSAGE];
-					else
+					// Skip if same msg
+					// but don't merge same messages if timestamp difference is greater than precofigured (useful when filtering same messages)
+					$tsDiff = abs($szLastMessageTimestamp - $logArray["timereported"][EVTIME_TIMESTAMP]); //sign depends on direction
+					//echo "uID=$uID; duplicates ($duplicateCount); delta ts = $tsDiff; isDublicate=".($szLastMessage == $logArray[SYSLOG_MESSAGE])."<br>";
+					if ( $szLastMessage == $logArray[SYSLOG_MESSAGE] && ($tsDiff < $DuplicateRecordMaxTsDistance))
 					{
-						// Skip if same msg
-						if ( $szLastMessage == $logArray[SYSLOG_MESSAGE] )
-						{
-							// Set last mgr
-							$szLastMessage = $logArray[SYSLOG_MESSAGE];
+						$szLastMessageTimestamp = $logArray["timereported"][EVTIME_TIMESTAMP];
 
-							// --- Extra Loop to get the next entry!
-							do
+						// --- Extra Loop to get the next entry!
+						// FIXME 230122 right now ReadNext skips entries only from custom  msgparser (see: classes/logstreamdb.class.php:601)
+						do
+						{
+							$ret = $stream->ReadNext($uID, $logArray);
+							$duplicateCount++;
+						} while ( $ret == ERROR_MSG_SKIPMESSAGE );
+						// --- 
+
+						// Skip entry
+						continue;
+					}else{
+						//inject entry about suppressed records
+						// FIXME any better way of doing that?
+						if($duplicateCount > 1){
+							foreach($content['Columns'] as $mycolkey)
 							{
-								$ret = $stream->ReadNext($uID, $logArray);
-							} while ( $ret == ERROR_MSG_SKIPMESSAGE );
-							// --- 
-
-							// Skip entry
-							continue;
+								if ( isset($fields[$mycolkey]) ) 
+								{	
+									$content['syslogmessages'][$counter]['values'][$mycolkey]['FieldColumn'] = $mycolkey;
+									$content['syslogmessages'][$counter]['values'][$mycolkey]['uid'] = $uID;
+									$content['syslogmessages'][$counter]['values'][$mycolkey]['FieldAlign'] = $fields[$mycolkey]['FieldAlign'];
+									$content['syslogmessages'][$counter]['values'][$mycolkey]['fieldcssclass'] = $content['syslogmessages'][$counter]['cssclass'];
+									$content['syslogmessages'][$counter]['values'][$mycolkey]['isnowrap'] = "nowrap";
+									$content['syslogmessages'][$counter]['values'][$mycolkey]['hasdetails'] = "false";
+									$content['syslogmessages'][$counter]['values'][$mycolkey]['detailimagealign'] = "TOP";
+
+									// Set default link 
+									$content['syslogmessages'][$counter]['values'][$mycolkey]['detaillink'] = "#";
+
+									if($content['fields'][$mycolkey]['FieldType'] == FILTER_TYPE_STRING && $mycolkey == SYSLOG_MESSAGE){
+										$content['syslogmessages'][$counter]['values'][$mycolkey]['fieldvalue'] = "... suppressed ".($duplicateCount)." duplicate(s)...";
+										//$content['syslogmessages'][$counter]['values'][$mycolkey]['rawfieldvalue'] = "rawfield";      // helper variable used for Popups!
+										//$content['syslogmessages'][$counter]['values'][$mycolkey]['encodedfieldvalue'] = "encoded field"; // Convert into filter format for submenus
+										$content['syslogmessages'][$counter]['values'][$mycolkey]['ismessagefield'] = false; //don't enable as it's not a real log message
+										$content['syslogmessages'][$counter]['values'][$mycolkey]['isnowrap'] = "";
+
+									}else	$content['syslogmessages'][$counter]['values'][$mycolkey]['fieldvalue'] = "";
+								}
+							}
+							//if enabled, then print it. otherwise this message wll be printed in wrong column!
+							$content['syslogmessages'][$counter]['MiscShowDebugGridCounter'] = $content['MiscShowDebugGridCounter'];
+							$counter++;
+							$duplicateCountTotal += $duplicateCount;
+							$duplicateCount = 0; //reset suppress counter
 						}
+						$szLastMessage = $logArray[SYSLOG_MESSAGE];
+						$szLastMessageTimestamp = $logArray["timereported"][EVTIME_TIMESTAMP];
 					}
-				}
+					$content['main_suppressed_recordcount'] = $duplicateCountTotal;
+				}// --- End of suppressing
 				// --- 
 
 				// --- Set CSS Class
@@ -689,6 +732,7 @@
 				} while ( $ret == ERROR_MSG_SKIPMESSAGE );
 				// --- 
 			} while ( $counter < $content['CurrentViewEntriesPerPage'] && ($ret == SUCCESS) );
+
 //print_r ( $content['syslogmessages'] );
 
 			// Move below processing - Read First and LAST UID's before start reading the stream!

src/lang/en/main.php

@@ -38,6 +38,7 @@
 $content['LN_GEN_NEXTPAGE'] = "Next Page";
 $content['LN_GEN_PREVIOUSPAGE'] = "Previous Page";
 $content['LN_GEN_RECORDCOUNT'] = "Total records found";
+$content['LN_GEN_SUPPRESSEDRECORDCOUNT'] = "Suppressed";
 $content['LN_GEN_PAGERSIZE'] = "Records per page";
 $content['LN_GEN_PAGE'] = "Page";
 $content['LN_GEN_PREDEFINEDSEARCHES'] = "Predefined Searches";

src/templates/index.html

@@ -171,6 +171,11 @@
 <!-- IF main_recordcount_found="true" -->
 	<td nowrap class="cellmenu2">{LN_GEN_RECORDCOUNT}:</td>
 	<td nowrap class="line2" STYLE="padding: 0px 15px 0px 0px;"><B>{main_recordcount}</B></td>
+
+	<!-- IF SUPPRESS_ENABLED="true" -->
+        	<td nowrap class="cellmenu2">{LN_GEN_SUPPRESSEDRECORDCOUNT}:</td>
+        	<td nowrap class="line2" STYLE="padding: 0px 15px 0px 0px;"><B>{main_suppressed_recordcount}</B></td>
+	<!-- ENDIF SUPPRESS_ENABLED="true" -->
 <!-- ENDIF main_recordcount_found="true" -->
 
 <!-- IF main_pagerenabled="true" -->