diff --git a/README.md b/README.md index 203124a..e2bb465 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,19 @@ Adiscon LogAnalyzer, a web frontend to log data from the same folks the created rsyslog # todo - - export: add checkbox to export full filtered history (now exports only current page) - - export: place ts into export filename (range from-to) - - BUG: "Suppress duplicated messages" doesn't work - - filter: allow to specify AFTER:n BEFORE:n (if possible/fast to implement) <- include records before and after match - - export: configure columns for file export (allow to remove unnecessary columns) <- exclude list of columns - BUG: sometimes spinner on index page is drawn in the middle of page irrespective of it's size, but should be drawn in the middle of screen - + - GUI: add checkbox on events page to suppress duplicates in export (override default ExportSuppressDuplicatedMessages + - GUI: add checkbox ... to override ExportAllMatchPages + +# changes 230122 + - export: append exported range timestamp to filename + - filter: dateto/datefrom add support for offset from current date, i.e. datefrom:T00:00:00 dateto:T01:00:00, datefrom:\-2T, datefrom:\-1T01 + - fixed BUG: "Suppress duplicated messages" + add property to control distance between duplicates + - GUI: if suppress is enabled, then show how much records were suppressed (LN_GEN_SUPPRESSEDRECORDCOUNT) + - $CFG[ExportSuppressDuplicatedMessages] - separate export suppressing from view + - $CFG[ExportAllMatchPages] - allow to control how much to export by default + - $CFG[DuplicateRecordMaxTsDistance] - check duplicates timestamp before suppressing, i.e. same message with 24h period may not be assumed as duplicate + # changes 230121 - datelastx - keep for backward compatibility (for saved searches); add datelastxx @@ -43,8 +49,10 @@ Adiscon LogAnalyzer, a web frontend to log data from the same folks the created - gui: add loglevel style colors and change color for full line; - filter: change datelastx behaviour - use number as hours indicator, i.e. datelastx:3 is 3 hours limit - #obsolete - - filter: allow to OR msg, i.e. key1 &key2 |key3; +# obsolete + - filter: allow to specify AFTER:n BEFORE:n (if possible/fast to implement) <- include records before and after match + - export: configure columns for file export (allow to remove unnecessary columns) <- exclude list of columns + - filter: allow to OR msg, i.e. key1 &key2 |key3; (instead use regex) - filter: date{from,to} - allow to use today/yesterday + short time, i.e. today 1h same as 1h, yesterday 2h, since/after/before/etc. - "Maximize view" - reloads page and resets search filter, hide toolbars with js instead - changing "Autoreload" does the same as "Max. view" diff --git a/src/export.php b/src/export.php index 973c0b4..d854e84 100644 --- a/src/export.php +++ b/src/export.php @@ -75,6 +75,8 @@ $content['searchstr'] = ""; $content['error_occured'] = false; +$content['ExportAllMatchPages'] = GetConfigSetting("ExportAllMatchPages", 0, CFGLEVEL_USER) == 1; +$content['SuppressDuplicatedMessages'] = GetConfigSetting("SuppressDuplicatedMessages", 0, CFGLEVEL_USER) == 1; // Check required input parameters if ( (isset($_GET['op']) && $_GET['op'] == "export") && @@ -203,32 +205,65 @@ if ( $ret == SUCCESS ) { //Loop through the messages! + $duplicateCount = 0; //FIXME while readNext record doesnt properly return skip_status keep it outside of loop body + $szLastMessageTimestamp = 0; + $DuplicateRecordMaxTsDistance = GetConfigSetting("DuplicateRecordMaxTsDistance", PHP_INT_MAX, CFGLEVEL_USER); do { // --- Extra stuff for suppressing messages if ( - GetConfigSetting("SuppressDuplicatedMessages", 0, CFGLEVEL_USER) == 1 - && - isset($logArray[SYSLOG_MESSAGE]) + $content['SuppressDuplicatedMessages'] && isset($logArray[SYSLOG_MESSAGE]) ) { - - if ( !isset($szLastMessage) ) // Only set lastmgr - $szLastMessage = $logArray[SYSLOG_MESSAGE]; - else + // Skip if same msg + // but don't merge same messages if timestamp difference is greater than precofigured (useful when filtering same messages) + $tsDiff = abs($szLastMessageTimestamp - $logArray["timereported"][EVTIME_TIMESTAMP]); + //echo "uID=$uID; duplicates ($duplicateCount); delta ts = $tsDiff; isDublicate=".($szLastMessage == $logArray[SYSLOG_MESSAGE])."
\n"; + if ( $szLastMessage == $logArray[SYSLOG_MESSAGE] && ($tsDiff < $DuplicateRecordMaxTsDistance)) { - // Skip if same msg - if ( $szLastMessage == $logArray[SYSLOG_MESSAGE] ) + $szLastMessageTimestamp = $logArray["timereported"][EVTIME_TIMESTAMP]; + + // --- Extra Loop to get the next entry! + // FIXME 230122 right now ReadNext skips entries only from custom msgparser (see: classes/logstreamdb.class.php:601) + do { - // Set last mgr - $szLastMessage = $logArray[SYSLOG_MESSAGE]; + $ret = $stream->ReadNext($uID, $logArray); + $duplicateCount++; + } while ( $ret == ERROR_MSG_SKIPMESSAGE ); + // --- + + // Skip entry + continue; + }else{ + //inject entry about suppressed records + // FIXME any better way of doing that? + if($duplicateCount > 1){ //ignore if only 1 duplicate + foreach($content['Columns'] as $mycolkey) + { + if ( isset($fields[$mycolkey]) ) + { + $content['syslogmessages'][$counter][$mycolkey]['FieldColumn'] = $mycolkey; + $content['syslogmessages'][$counter][$mycolkey]['uid'] = $uID; - // Skip entry - continue; + if($content['fields'][$mycolkey]['FieldType'] == FILTER_TYPE_STRING && $mycolkey == SYSLOG_MESSAGE){ + $content['syslogmessages'][$counter][$mycolkey]['fieldvalue'] = "... suppressed $duplicateCount duplicate(s)..."; + }else $content['syslogmessages'][$counter][$mycolkey]['fieldvalue'] = "\t"; + } + } + $counter++; + $duplicateCount = 0; //reset suppress counter } + $szLastMessage = $logArray[SYSLOG_MESSAGE]; + $szLastMessageTimestamp = $logArray["timereported"][EVTIME_TIMESTAMP]; } } // --- + + if(!isset($content['period_start_ts'])){ //store first record ts + $content['period_start_ts'] = $logArray["timereported"][EVTIME_TIMESTAMP]; + }//*else if(!isset($content['period_start_ts'])){ // FIXME store last record ts + $content['period_end_ts'] = $logArray["timereported"][EVTIME_TIMESTAMP]; + //} // --- Now we populate the values array! foreach($content['Columns'] as $mycolkey) @@ -286,7 +321,17 @@ // Increment Counter $counter++; - } while ($counter < $content['CurrentViewEntriesPerPage'] && ($ret = $stream->ReadNext($uID, $logArray)) == SUCCESS); + + // --- Extra Loop to get the next entry! + do + { + $ret = $stream->ReadNext($uID, $logArray); + } while ( $ret == ERROR_MSG_SKIPMESSAGE ); + + if(!$content['ExportAllMatchPages'] && $counter >= $content['CurrentViewEntriesPerPage']){ + break; + } + } while ($ret == SUCCESS); if ( $content['read_direction'] == EnumReadDirection::Forward ) { @@ -331,7 +376,7 @@ $szOutputMimeType = "text/plain"; $szOutputCharset = ""; - $szOutputFileName = "ExportMessages"; + $szOutputFileName = "ExportMessages_".date('Ymd\THis',$content['period_start_ts'])."-".date('Ymd\THis',$content['period_end_ts']); $szOutputFileExtension = ".txt"; $szOPFieldSeparator = " "; $szOPFirstLineFieldNames = true; diff --git a/src/include/config.sample.php b/src/include/config.sample.php index 9dfd00f..7488924 100644 --- a/src/include/config.sample.php +++ b/src/include/config.sample.php @@ -82,6 +82,8 @@ // --- Default Export options $CFG['ExportUseTodayYesterday'] = 0; // Same as ViewUseTodayYesterday. By default export normal dates +$CFG['ExportSuppressDuplicatedMessages'] = 0 // If enabled, then export will contain no duplicates (@see related DuplicateRecordMaxTsDistance) +$CFG['ExportAllMatchPages'] = 0 // By default export only selected page results // --- Default Frontend Options $CFG['PrependTitle'] = ""; // If set, this text will be prepended withint the title tag @@ -104,6 +106,8 @@ $CFG['EnableContextLinks'] = 1; // if enabled, context links within the messages will automatically be created and added. Set this to 0 to disable all context links. $CFG['EnableIPAddressResolve'] = 1; // If enabled, IP Addresses inline messages are automatically resolved and the result is added in brackets {} behind the IP Address $CFG['SuppressDuplicatedMessages'] = 0; // If enabled, duplicated messages will be suppressed in the main display. +$CFG['DuplicateRecordMaxTsDistance'] = PHP_INT_MAX; // Max timestamp delta between two matching records. If delta is less than this value then records will be suppressed + $CFG['TreatNotFoundFiltersAsTrue'] = 0; // If you filter / search for messages, and the fields you are filtering for is not found, the filter result is treaten as TRUE! $CFG['PopupMenuTimeout'] = 3000; // This variable defines the default timeout value for popup menus in milliseconds. (those menus which popup when you click on the value of a field. $CFG['PhplogconLogoUrl'] = ""; // Put an Url to a custom toplogo you want to use. diff --git a/src/include/functions_common.php b/src/include/functions_common.php index 5339116..e469eaf 100644 --- a/src/include/functions_common.php +++ b/src/include/functions_common.php @@ -61,7 +61,7 @@ // Try to enable a little more memory in case we do some more filtering @ini_set('memory_limit', '512M'); -// --- +// --- // Default language $LANG_EN = "en"; // Used for fallback @@ -1317,6 +1317,9 @@ function RedirectResult( $szMsg, $newpage ) */ function GetEventTime($szTimStr) { + //remove field extra escaping + $szTimStr = str_replace("\\","",$szTimStr); + // Sample: Mar 10 14:45:44 if ( preg_match("/(...) ([0-9]{1,2}) ([0-9]{1,2}):([0-9]{1,2}):([0-9]{1,2})/", $szTimStr, $out ) ) { @@ -1393,6 +1396,23 @@ function GetEventTime($szTimStr) $eventtime[EVTIME_TIMEZONE] = date('O'); // Get default Offset $eventtime[EVTIME_MICROSECONDS] = 0; } + // Sample: \-1T11:10:50 or \-1T0:1 or T12:0, etc + else if ( preg_match("/(-?[0-9]{1,2})?T([0-9]{0,2}):?([0-9]{0,2}):?([0-9]{0,2})/", $szTimStr, $out ) ) + { + $hh = is_numeric($out[2]) ? $out[2] : 0; + $mm = is_numeric($out[3]) ? $out[3] : 0; + $ss = is_numeric($out[4]) ? $out[4] : 0; + + // RFC 3164 typical timestamp + $szTime = mktime($hh, $mm, $ss); + if(is_numeric($out[1]) && $out[1] < 0){//day offset + $szTime = strtotime($out[1].' days', $szTime); + } + + $eventtime[EVTIME_TIMESTAMP] = $szTime; + $eventtime[EVTIME_TIMEZONE] = date('O'); // Get default Offset + $eventtime[EVTIME_MICROSECONDS] = 0; + } else { $eventtime[EVTIME_TIMESTAMP] = 0; @@ -1733,8 +1753,8 @@ function StartPHPSession() global $RUNMODE; if ( $RUNMODE == RUNMODE_WEBSERVER ) { - // This will start the session - @session_start(); + // This will start the session + @session_start(); if ( !isset($_SESSION['SESSION_STARTED']) ) diff --git a/src/include/functions_filters.php b/src/include/functions_filters.php index 6524e09..1037319 100644 --- a/src/include/functions_filters.php +++ b/src/include/functions_filters.php @@ -323,6 +323,9 @@ function GetMessageTypeDisplayName( $nMsgTypeID ) function GetTimeStampFromTimeString($szTimeString) { + //remove field extra escaping + $szTimeString = str_replace('\\', '', $szTimeString); + //Sample: 2008-4-1T00:00:00 if ( preg_match("/([0-9]{4,4})-([0-9]{1,2})-([0-9]{1,2})T([0-9]{1,2}):([0-9]{1,2}):([0-9]{1,2})$/", $szTimeString, $out) ) { @@ -335,6 +338,20 @@ function GetTimeStampFromTimeString($szTimeString) // return new timestamp return mktime(0,0,0, $out[2], $out[3], $out[1]); } + // Sample: \-1T11:10:50 or \-1T0:1 or T12:0, etc + else if ( preg_match("/(-?[0-9]{1,2})?T([0-9]{0,2}):?([0-9]{0,2}):?([0-9]{0,2})/", $szTimeString, $out ) ) + { + $hh = is_numeric($out[2]) ? $out[2] : 0; + $mm = is_numeric($out[3]) ? $out[3] : 0; + $ss = is_numeric($out[4]) ? $out[4] : 0; + + $szTime = mktime($hh, $mm, $ss); + if(is_numeric($out[1]) && $out[1] < 0){//day offset + $szTime = strtotime($out[1].' days', $szTime); + } + + return $szTime; + } else { OutputDebugMessage("Unparseable Time in GetTimeStampFromTimeString - '" . $szTimeString . "'", DEBUG_WARN); diff --git a/src/index.php b/src/index.php index d182491..6a7e716 100644 --- a/src/index.php +++ b/src/index.php @@ -105,6 +105,9 @@ $content['skipone'] = false; // --- +// Init show suppressed count flag +$content['SUPPRESS_ENABLED'] = GetConfigSetting("SuppressDuplicatedMessages", 0, CFGLEVEL_USER) == 1; + // Init Export Stuff! $content['EXPORT_ENABLED'] = true; @@ -336,38 +339,78 @@ // --- //Loop through the messages! + $duplicateCountTotal = 0; + $duplicateCount = 0; //FIXME while readNext record doesnt properly return skip_status keep it outside of loop body + $szLastMessageTimestamp = 0; + $DuplicateRecordMaxTsDistance = GetConfigSetting("DuplicateRecordMaxTsDistance", PHP_INT_MAX, CFGLEVEL_USER); do { // --- Extra stuff for suppressing messages if ( - GetConfigSetting("SuppressDuplicatedMessages", 0, CFGLEVEL_USER) == 1 - && - isset($logArray[SYSLOG_MESSAGE]) - ) + GetConfigSetting("SuppressDuplicatedMessages", 0, CFGLEVEL_USER) == 1 + && + isset($logArray[SYSLOG_MESSAGE]) + ) { - - if ( !isset($szLastMessage) ) // Only set lastmgr - $szLastMessage = $logArray[SYSLOG_MESSAGE]; - else + // Skip if same msg + // but don't merge same messages if timestamp difference is greater than precofigured (useful when filtering same messages) + $tsDiff = abs($szLastMessageTimestamp - $logArray["timereported"][EVTIME_TIMESTAMP]); //sign depends on direction + //echo "uID=$uID; duplicates ($duplicateCount); delta ts = $tsDiff; isDublicate=".($szLastMessage == $logArray[SYSLOG_MESSAGE])."
"; + if ( $szLastMessage == $logArray[SYSLOG_MESSAGE] && ($tsDiff < $DuplicateRecordMaxTsDistance)) { - // Skip if same msg - if ( $szLastMessage == $logArray[SYSLOG_MESSAGE] ) - { - // Set last mgr - $szLastMessage = $logArray[SYSLOG_MESSAGE]; + $szLastMessageTimestamp = $logArray["timereported"][EVTIME_TIMESTAMP]; - // --- Extra Loop to get the next entry! - do + // --- Extra Loop to get the next entry! + // FIXME 230122 right now ReadNext skips entries only from custom msgparser (see: classes/logstreamdb.class.php:601) + do + { + $ret = $stream->ReadNext($uID, $logArray); + $duplicateCount++; + } while ( $ret == ERROR_MSG_SKIPMESSAGE ); + // --- + + // Skip entry + continue; + }else{ + //inject entry about suppressed records + // FIXME any better way of doing that? + if($duplicateCount > 1){ + foreach($content['Columns'] as $mycolkey) { - $ret = $stream->ReadNext($uID, $logArray); - } while ( $ret == ERROR_MSG_SKIPMESSAGE ); - // --- - - // Skip entry - continue; + if ( isset($fields[$mycolkey]) ) + { + $content['syslogmessages'][$counter]['values'][$mycolkey]['FieldColumn'] = $mycolkey; + $content['syslogmessages'][$counter]['values'][$mycolkey]['uid'] = $uID; + $content['syslogmessages'][$counter]['values'][$mycolkey]['FieldAlign'] = $fields[$mycolkey]['FieldAlign']; + $content['syslogmessages'][$counter]['values'][$mycolkey]['fieldcssclass'] = $content['syslogmessages'][$counter]['cssclass']; + $content['syslogmessages'][$counter]['values'][$mycolkey]['isnowrap'] = "nowrap"; + $content['syslogmessages'][$counter]['values'][$mycolkey]['hasdetails'] = "false"; + $content['syslogmessages'][$counter]['values'][$mycolkey]['detailimagealign'] = "TOP"; + + // Set default link + $content['syslogmessages'][$counter]['values'][$mycolkey]['detaillink'] = "#"; + + if($content['fields'][$mycolkey]['FieldType'] == FILTER_TYPE_STRING && $mycolkey == SYSLOG_MESSAGE){ + $content['syslogmessages'][$counter]['values'][$mycolkey]['fieldvalue'] = "... suppressed ".($duplicateCount)." duplicate(s)..."; + //$content['syslogmessages'][$counter]['values'][$mycolkey]['rawfieldvalue'] = "rawfield"; // helper variable used for Popups! + //$content['syslogmessages'][$counter]['values'][$mycolkey]['encodedfieldvalue'] = "encoded field"; // Convert into filter format for submenus + $content['syslogmessages'][$counter]['values'][$mycolkey]['ismessagefield'] = false; //don't enable as it's not a real log message + $content['syslogmessages'][$counter]['values'][$mycolkey]['isnowrap'] = ""; + + }else $content['syslogmessages'][$counter]['values'][$mycolkey]['fieldvalue'] = ""; + } + } + //if enabled, then print it. otherwise this message wll be printed in wrong column! + $content['syslogmessages'][$counter]['MiscShowDebugGridCounter'] = $content['MiscShowDebugGridCounter']; + $counter++; + $duplicateCountTotal += $duplicateCount; + $duplicateCount = 0; //reset suppress counter } + $szLastMessage = $logArray[SYSLOG_MESSAGE]; + $szLastMessageTimestamp = $logArray["timereported"][EVTIME_TIMESTAMP]; } - } + $content['main_suppressed_recordcount'] = $duplicateCountTotal; + }// --- End of suppressing // --- // --- Set CSS Class @@ -689,6 +732,7 @@ } while ( $ret == ERROR_MSG_SKIPMESSAGE ); // --- } while ( $counter < $content['CurrentViewEntriesPerPage'] && ($ret == SUCCESS) ); + //print_r ( $content['syslogmessages'] ); // Move below processing - Read First and LAST UID's before start reading the stream! diff --git a/src/lang/en/main.php b/src/lang/en/main.php index 2fd9d6e..ba157cd 100644 --- a/src/lang/en/main.php +++ b/src/lang/en/main.php @@ -38,6 +38,7 @@ $content['LN_GEN_NEXTPAGE'] = "Next Page"; $content['LN_GEN_PREVIOUSPAGE'] = "Previous Page"; $content['LN_GEN_RECORDCOUNT'] = "Total records found"; +$content['LN_GEN_SUPPRESSEDRECORDCOUNT'] = "Suppressed"; $content['LN_GEN_PAGERSIZE'] = "Records per page"; $content['LN_GEN_PAGE'] = "Page"; $content['LN_GEN_PREDEFINEDSEARCHES'] = "Predefined Searches"; diff --git a/src/templates/index.html b/src/templates/index.html index 862ab23..b4950dc 100644 --- a/src/templates/index.html +++ b/src/templates/index.html @@ -171,6 +171,11 @@ {LN_GEN_RECORDCOUNT}: {main_recordcount} + + + {LN_GEN_SUPPRESSEDRECORDCOUNT}: + {main_suppressed_recordcount} +