[389-commits] Branch '389-ds-base-1.3.0' - 16 commits - ldap/admin ldap/servers man/man1

ldap/admin/src/logconv.pl | 3168 +++++++++++++++-------------- ldap/servers/slapd/back-ldbm/ldbm_search.c | 6 ldap/servers/slapd/result.c | 3 ldap/servers/slapd/slapi-plugin.h | 1 man/man1/logconv.pl.1 | 44 5 files changed, 1716 insertions(+), 1506 deletions(-) New commits: commit 9103b3e3812fd2cd05f281729e017c46f1c841e5 Author: Rich Megginson <rmeggins(a)redhat.com&gt; Date: Fri Sep 27 11:08:29 2013 -0600 Ticket 47533 logconv: some stats do not work across server restarts https://fedorahosted.org/389/ticket/47533 Reviewed by: mreynolds (Thanks!) Branch: master Fix Description: Most of the stat hashes use connid,opid as the hash key. The connid,opid are not unique because the server recycles connection ids when the server restarts. The solution is to use a triplet of $serverRestartCount,$connid,$opid as the hash key. This uniquely identifies a particular log event over restarts. There is also some additional cleanup that needs to be done when the server is reset - returning fds, marking connections as closed, etc. Also fixed some gmtime handling - we were doing too many conversions. This fix adds another command line option - -U - for a summary of search filters which are unindexed or have unindexed components. The full detailed report is just too much data. Platforms tested: RHEL6 x86_64 Flag Day: no Doc impact: no (cherry picked from commit 92ac8f48c1e5a8a049f4cfd1912fd570c628408f) (cherry picked from commit e38685be139c2420cc969499324405a30203122a) diff --git a/ldap/admin/src/logconv.pl b/ldap/admin/src/logconv.pl index ca07a3a..275ce34 100755 --- a/ldap/admin/src/logconv.pl +++ b/ldap/admin/src/logconv.pl @@ -86,7 +86,8 @@ my $xi = 0; my $bindReportDN; my $usage = ""; my @latency; -my @openConnection; +# key is conn number - val is IP address +my %openConnection; my @errorCode; my @errtext; my @errornum; @@ -133,7 +134,8 @@ GetOptions( 'u' => sub { $usage = $usage . "u"; }, 'r' => sub { $usage = $usage . "r"; }, 'y' => sub { $usage = $usage . "y"; }, - 'p' => sub { $usage = $usage . "p"; } + 'p' => sub { $usage = $usage . "p"; }, + 'U' => sub { $usage = $usage . "U"; } ); # @@ -657,7 +659,7 @@ print "Paged Searches: $pagedSearchCount\n"; print "Unindexed Searches: $unindexedSrchCountNotesA\n"; print "Unindexed Components: $unindexedSrchCountNotesU\n"; -if ($verb eq "yes" || $usage =~ /u/){ +if ($verb eq "yes" || $usage =~ /u/ || $usage =~ /U/){ if ($unindexedSrchCountNotesA > 0){ my $conn_hash = $hashes->{conn_hash}; my $notesa_conn_op = $hashes->{notesa_conn_op}; @@ -670,28 +672,39 @@ if ($verb eq "yes" || $usage =~ /u/){ my $notesCount = 1; my $unindexedIp; - while (my ($conn_op, $count) = each %{$notesa_conn_op}) { - my ($conn, $op) = split(",", $conn_op); - if (exists($conn_hash->{$conn}) && defined($conn_hash->{$conn})) { - $unindexedIp = $conn_hash->{$conn}; - } else { - $unindexedIp = "?"; - } - print "\n Unindexed Components #".$notesCount."\n"; $notesCount++; - print " - Date/Time: $time_conn_op->{$conn_op}\n"; - print " - Connection Number: $conn\n"; - print " - Operation Number: $op\n"; - print " - Etime: $etime_conn_op->{$conn_op}\n"; - print " - Nentries: $nentries_conn_op->{$conn_op}\n"; - print " - IP Address: $unindexedIp\n"; - if (exists($base_conn_op->{$conn_op}) && defined($base_conn_op->{$conn_op})) { - print " - Search Base: $base_conn_op->{$conn_op}\n"; + my %uniqFilt = (); # hash of unique filters + while (my ($srcnt_conn_op, $count) = each %{$notesa_conn_op}) { + my ($srvRstCnt, $conn, $op) = split(",", $srcnt_conn_op); + $unindexedIp = getIPfromConn($conn, $srvRstCnt); + if ($usage =~ /u/) { + print "\n Unindexed Search #".$notesCount."\n"; + print " - Date/Time: $time_conn_op->{$srcnt_conn_op}\n"; + print " - Connection Number: $conn\n"; + print " - Operation Number: $op\n"; + print " - Etime: $etime_conn_op->{$srcnt_conn_op}\n"; + print " - Nentries: $nentries_conn_op->{$srcnt_conn_op}\n"; + print " - IP Address: $unindexedIp\n"; + if (exists($base_conn_op->{$srcnt_conn_op}) && defined($base_conn_op->{$srcnt_conn_op})) { + print " - Search Base: $base_conn_op->{$srcnt_conn_op}\n"; + } + if (exists($scope_conn_op->{$srcnt_conn_op}) && defined($scope_conn_op->{$srcnt_conn_op})) { + print " - Search Scope: $scope_conn_op->{$srcnt_conn_op}\n"; + } } - if (exists($scope_conn_op->{$conn_op}) && defined($scope_conn_op->{$conn_op})) { - print " - Search Scope: $scope_conn_op->{$conn_op}\n"; + if (exists($filter_conn_op->{$srcnt_conn_op}) && defined($filter_conn_op->{$srcnt_conn_op})) { + if ($usage =~ /u/) { + print " - Search Filter: $filter_conn_op->{$srcnt_conn_op}\n"; + } + $uniqFilt{$filter_conn_op->{$srcnt_conn_op}}++; } - if (exists($filter_conn_op->{$conn_op}) && defined($filter_conn_op->{$conn_op})) { - print " - Search Filter: $filter_conn_op->{$conn_op}\n"; + $notesCount++; + } + if ($usage =~ /U/) { + print "\n Unindexed Search Summary - $notesCount total unindexed searches\n"; + foreach my $key (sort { $uniqFilt{$b} <=> $uniqFilt{$a} } keys %uniqFilt) { + if ($uniqFilt{$key} > 0) { + printf " - Number of times used unindexed: %10d Filter: $key\n", $uniqFilt{$key}; + } } } } @@ -707,28 +720,39 @@ if ($verb eq "yes" || $usage =~ /u/){ my $notesCount = 1; my $unindexedIp; - while (my ($conn_op, $count) = each %{$notesu_conn_op}) { - my ($conn, $op) = split(",", $conn_op); - if (exists($conn_hash->{$conn}) && defined($conn_hash->{$conn})) { - $unindexedIp = $conn_hash->{$conn}; - } else { - $unindexedIp = "?"; - } - print "\n Unindexed Components #".$notesCount."\n"; $notesCount++; - print " - Date/Time: $time_conn_op->{$conn_op}\n"; - print " - Connection Number: $conn\n"; - print " - Operation Number: $op\n"; - print " - Etime: $etime_conn_op->{$conn_op}\n"; - print " - Nentries: $nentries_conn_op->{$conn_op}\n"; - print " - IP Address: $unindexedIp\n"; - if (exists($base_conn_op->{$conn_op}) && defined($base_conn_op->{$conn_op})) { - print " - Search Base: $base_conn_op->{$conn_op}\n"; + my %uniqFilt = (); # hash of unique filters + while (my ($srcnt_conn_op, $count) = each %{$notesu_conn_op}) { + my ($srvRstCnt, $conn, $op) = split(",", $srcnt_conn_op); + $unindexedIp = getIPfromConn($conn, $srvRstCnt); + if ($usage =~ /u/) { + print "\n Unindexed Component #".$notesCount."\n"; + print " - Date/Time: $time_conn_op->{$srcnt_conn_op}\n"; + print " - Connection Number: $conn\n"; + print " - Operation Number: $op\n"; + print " - Etime: $etime_conn_op->{$srcnt_conn_op}\n"; + print " - Nentries: $nentries_conn_op->{$srcnt_conn_op}\n"; + print " - IP Address: $unindexedIp\n"; + if (exists($base_conn_op->{$srcnt_conn_op}) && defined($base_conn_op->{$srcnt_conn_op})) { + print " - Search Base: $base_conn_op->{$srcnt_conn_op}\n"; + } + if (exists($scope_conn_op->{$srcnt_conn_op}) && defined($scope_conn_op->{$srcnt_conn_op})) { + print " - Search Scope: $scope_conn_op->{$srcnt_conn_op}\n"; + } } - if (exists($scope_conn_op->{$conn_op}) && defined($scope_conn_op->{$conn_op})) { - print " - Search Scope: $scope_conn_op->{$conn_op}\n"; + if (exists($filter_conn_op->{$srcnt_conn_op}) && defined($filter_conn_op->{$srcnt_conn_op})) { + if ($usage =~ /u/) { + print " - Search Filter: $filter_conn_op->{$srcnt_conn_op}\n"; + } + $uniqFilt{$filter_conn_op->{$srcnt_conn_op}}++; } - if (exists($filter_conn_op->{$conn_op}) && defined($filter_conn_op->{$conn_op})) { - print " - Search Filter: $filter_conn_op->{$conn_op}\n"; + $notesCount++; + } + if ($usage =~ /U/) { + print "\n Unindexed Component Summary - $notesCount total unindexed components\n"; + foreach my $key (sort { $uniqFilt{$b} <=> $uniqFilt{$a} } keys %uniqFilt) { + if ($uniqFilt{$key} > 0) { + printf " - Number of times used unindexed: %10d Filter: $key\n", $uniqFilt{$key}; + } } } } @@ -829,12 +853,10 @@ if ($verb eq "yes" || $usage =~ /y/){ ################################### if ($verb eq "yes" || $usage =~ /p/){ - if (@openConnection > 0){ + if (%openConnection){ print "\n\n----- Current Open Connection IDs ----- \n\n"; - for (my $i=0; $i <= $#openConnection ; $i++) { - if ($openConnection[$i]) { - print "Conn Number: $i (" . getIPfromConn($i) . ")\n"; - } + while (my ($connid, $ip) = each %openConnection) { + print "Conn Number: $connid ($ip)\n"; } } } @@ -904,13 +926,13 @@ if ($verb eq "yes" || $usage =~ /f/ ){ my $badpwd_conn_op = $hashes->{badpwd_conn_op}; # key is binddn - val is count my %badPassword = (); - my @badPasswordIp = (); - while (my ($conn_op, $count) = each %{$badpwd_conn_op}) { - my ($conn, $op) = split(",", $conn_op); - if (exists($bind_conn_op->{$conn_op}) && defined($bind_conn_op->{$conn_op})) { - my $binddn = $bind_conn_op->{$conn_op}; + my %badPasswordIp = (); + while (my ($srcnt_conn_op, $ip) = each %{$badpwd_conn_op}) { + my ($srvRstCnt, $conn, $op) = split(",", $srcnt_conn_op); + if (exists($bind_conn_op->{$srcnt_conn_op}) && defined($bind_conn_op->{$srcnt_conn_op})) { + my $binddn = $bind_conn_op->{$srcnt_conn_op}; $badPassword{$binddn}++; - push @badPasswordIp, getIPfromConn($conn); + $badPasswordIp{$ip}++; } } # sort the new hash of $badPassword{} @@ -923,8 +945,11 @@ if ($verb eq "yes" || $usage =~ /f/ ){ printf "%-4s %-40s\n", $badPassword{"$badpw"}, $badpw; } print "\nFrom the IP address(s) :\n\n"; - for (my $i=0; $i<$badPwdCount; $i++) { - print "\t\t$badPasswordIp[$i]\n"; + $bpCount = 0; + foreach my $ip (sort {$badPassword{$b} <=> $badPassword{$a} } keys %badPasswordIp){ + if ($bpCount > $sizeCount){ last;} + $bpCount++; + printf "%-4s %-16s\n", $badPasswordIp{$ip}, $ip; } if ($bpTotal > $badPwdCount){ print "\n** Warning : Wrongly reported failed login attempts : ". ($bpTotal - $badPwdCount) . "\n"; @@ -1220,33 +1245,28 @@ if ($usage =~ /g/i || $verb eq "yes"){ print "\n\n----- Abandon Request Stats -----\n\n"; - while (my ($conn_op, $targ_msgid) = each %{$abandon_conn_op}) { - my ($conn, $op) = split(",", $conn_op); + while (my ($srcnt_conn_op, $targ_msgid) = each %{$abandon_conn_op}) { + my ($srvRstCnt, $conn, $op) = split(",", $srcnt_conn_op); my ($targetop, $msgid) = split(",", $targ_msgid); - my $conn_targ = "$conn,$targetop"; - my $clientIP; - if (exists($conn_hash->{$conn}) && defined($conn_hash->{$conn})) { - $clientIP = $conn_hash->{$conn}; - } else { - $clientIP = "Unknown"; - } - if (exists($srch_conn_op->{$conn_targ}) && defined($srch_conn_op->{$conn_targ})) { + my $srcnt_conn_targ = "$srvRstCnt,$conn,$targetop"; + my $clientIP = getIPfromConn($conn, $srvRstCnt); + if (exists($srch_conn_op->{$srcnt_conn_targ}) && defined($srch_conn_op->{$srcnt_conn_targ})) { print " - SRCH conn=$conn op=$targetop msgid=$msgid client=$clientIP\n"; - } elsif (exists($del_conn_op->{$conn_targ}) && defined($del_conn_op->{$conn_targ})) { + } elsif (exists($del_conn_op->{$srcnt_conn_targ}) && defined($del_conn_op->{$srcnt_conn_targ})) { print " - DEL conn=$conn op=$targetop msgid=$msgid client=$clientIP\n"; - } elsif (exists($add_conn_op->{$conn_targ}) && defined($add_conn_op->{$conn_targ})) { + } elsif (exists($add_conn_op->{$srcnt_conn_targ}) && defined($add_conn_op->{$srcnt_conn_targ})) { print " - ADD conn=$conn op=$targetop msgid=$msgid client=$clientIP\n"; - } elsif (exists($mod_conn_op->{$conn_targ}) && defined($mod_conn_op->{$conn_targ})) { + } elsif (exists($mod_conn_op->{$srcnt_conn_targ}) && defined($mod_conn_op->{$srcnt_conn_targ})) { print " - MOD conn=$conn op=$targetop msgid=$msgid client=$clientIP\n"; - } elsif (exists($cmp_conn_op->{$conn_targ}) && defined($cmp_conn_op->{$conn_targ})) { + } elsif (exists($cmp_conn_op->{$srcnt_conn_targ}) && defined($cmp_conn_op->{$srcnt_conn_targ})) { print " - CMP conn=$conn op=$targetop msgid=$msgid client=$clientIP\n"; - } elsif (exists($mdn_conn_op->{$conn_targ}) && defined($mdn_conn_op->{$conn_targ})) { + } elsif (exists($mdn_conn_op->{$srcnt_conn_targ}) && defined($mdn_conn_op->{$srcnt_conn_targ})) { print " - MODRDN conn=$conn op=$targetop msgid=$msgid client=$clientIP\n"; - } elsif (exists($bind_conn_op->{$conn_targ}) && defined($bind_conn_op->{$conn_targ})) { + } elsif (exists($bind_conn_op->{$srcnt_conn_targ}) && defined($bind_conn_op->{$srcnt_conn_targ})) { print " - BIND conn=$conn op=$targetop msgid=$msgid client=$clientIP\n"; - } elsif (exists($unbind_conn_op->{$conn_targ}) && defined($unbind_conn_op->{$conn_targ})) { + } elsif (exists($unbind_conn_op->{$srcnt_conn_targ}) && defined($unbind_conn_op->{$srcnt_conn_targ})) { print " - UNBIND conn=$conn op=$targetop msgid=$msgid client=$clientIP\n"; - } elsif (exists($ext_conn_op->{$conn_targ}) && defined($ext_conn_op->{$conn_targ})) { + } elsif (exists($ext_conn_op->{$srcnt_conn_targ}) && defined($ext_conn_op->{$srcnt_conn_targ})) { print " - EXT conn=$conn op=$targetop msgid=$msgid client=$clientIP\n"; } else { print " - UNKNOWN conn=$conn op=$targetop msgid=$msgid client=$clientIP\n"; @@ -1521,6 +1541,44 @@ processOpForBindReport } my ($last_tm, $lastzone, $last_min, $gmtime, $tzoff); +sub handleConnClose +{ + my $connID = shift; + $fdReturned++; + $simConnection--; + + delete $openConnection{$connID}; + if ($reportStats or ($verb eq "yes") || ($usage =~ /y/)) { + # if we didn't see the start time of this connection + # i.e. due to truncation or log rotation + # then just set to 0 + my $stoc = $hashes->{start_time_of_connection}->{$connID} || 0; + $hashes->{end_time_of_connection}->{$connID} = $gmtime || 0; + my $diff = $hashes->{end_time_of_connection}->{$connID} - $stoc; + $hashes->{start_time_of_connection}->{$connID} = $hashes->{end_time_of_connection}->{$connID} = 0; + if ($diff <= 1) { $latency[0]++;} + if ($diff == 2) { $latency[1]++;} + if ($diff == 3) { $latency[2]++;} + if ($diff >= 4 && $diff <=5 ) { $latency[3]++;} + if ($diff >= 6 && $diff <=10 ) { $latency[4]++;} + if ($diff >= 11 && $diff <=15 ) { $latency[5]++;} + if ($diff >= 16) { $latency[6] ++;} + } +} + +sub handleRestart +{ + # there are some stats which depend on restarts + if ($verb eq "yes") { + print "Found a restart - resetting stats that depend on restarts"; + } + for my $connID (keys %openConnection) { + handleConnClose($connID); + } + %openConnection = (); # reset open connections + $serverRestartCount++; +} + sub parseLineNormal { local $_ = $logline; @@ -1612,42 +1670,42 @@ sub parseLineNormal $anyAttrs++; } if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{srch_conn_op}->{"$1,$2"}++;} + if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{srch_conn_op}->{"$serverRestartCount,$1,$2"}++;} } } if (m/ DEL/){ $delCount++; if($reportStats){ inc_stats('del',$s_stats,$m_stats); } if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{del_conn_op}->{"$1,$2"}++;} + if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{del_conn_op}->{"$serverRestartCount,$1,$2"}++;} } } if (m/ MOD dn=/){ $modCount++; if($reportStats){ inc_stats('mod',$s_stats,$m_stats); } if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{mod_conn_op}->{"$1,$2"}++;} + if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{mod_conn_op}->{"$serverRestartCount,$1,$2"}++;} } } if (m/ ADD/){ $addCount++; if($reportStats){ inc_stats('add',$s_stats,$m_stats); } if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{add_conn_op}->{"$1,$2"}++;} + if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{add_conn_op}->{"$serverRestartCount,$1,$2"}++;} } } if (m/ MODRDN/){ $modrdnCount++; if($reportStats){ inc_stats('modrdn',$s_stats,$m_stats); } if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{mdn_conn_op}->{"$1,$2"}++;} + if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{mdn_conn_op}->{"$serverRestartCount,$1,$2"}++;} } } if (m/ CMP dn=/){ $cmpCount++; if($reportStats){ inc_stats('cmp',$s_stats,$m_stats); } if ($verb eq "yes" || $usage =~ /g/i){ - if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{cmp_conn_op}->{"$1,$2"}++;} + if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{cmp_conn_op}->{"$serverRestartCount,$1,$2"}++;} } } if (m/ ABANDON /){ @@ -1662,7 +1720,7 @@ sub parseLineNormal if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+) +ABANDON +targetop= *([0-9a-zA-Z]+) +msgid= *([0-9\-]+)/i ){ # abandon_conn_op - key is the conn,op of the ABANDON request # the value is the targetop,msgid of the ABANDON request - $hashes->{abandon_conn_op}->{"$1,$2"} = "$3,$4"; # targetop,msgid + $hashes->{abandon_conn_op}->{"$serverRestartCount,$1,$2"} = "$3,$4"; # targetop,msgid } } if (m/ VLV /){ @@ -1676,7 +1734,9 @@ sub parseLineNormal if (m/ SORT /){$vlvSortCount++} if (m/ version=2/){$v2BindCount++} if (m/ version=3/){$v3BindCount++} - if (m/ conn=1 fd=/){$serverRestartCount++} + if (m/ conn=1 fd=/){ + handleRestart(); + } if (m/ SSL connection from/){$sslCount++; if($reportStats){ inc_stats('sslconns',$s_stats,$m_stats); }} if (m/ connection from local to /){$ldapiCount++;} if($_ =~ /AUTOBIND dn=\"(.*)\"/){ @@ -1697,9 +1757,11 @@ sub parseLineNormal } } if (m/ connection from/){ + my $ip; if ($_ =~ /connection from *([0-9A-Fa-f\.\:]+)/i ){ + $ip = $1; for (my $xxx =0; $xxx < $#excludeIP; $xxx++){ - if ($excludeIP[$xxx] eq $1){$exc = "yes";} + if ($excludeIP[$xxx] eq $ip){$exc = "yes";} } if ($exc ne "yes"){ $connectionCount++; @@ -1711,40 +1773,17 @@ sub parseLineNormal $maxsimConnection = $simConnection; } ($connID) = $_ =~ /conn=(\d*)\s/; - $openConnection[$connID]++; + $openConnection{$connID} = $ip; if ($reportStats or ($verb eq "yes") || ($usage =~ /y/)) { - my ($time, $tzone) = split (' ', $_); - my ($date, $hr, $min, $sec) = split (':', $time); - my ($day, $mon, $yr) = split ('/', $date); - $day =~ s/\[//; - $hashes->{start_time_of_connection}->{$connID} = timegm($sec, $min, $hr, $day, $monthname{$mon}, $yr); + $hashes->{start_time_of_connection}->{$connID} = $gmtime; } } if (m/ SSL client bound as /){$sslClientBindCount++;} if (m/ SSL failed to map client certificate to LDAP DN/){$sslClientFailedCount++;} if (m/ fd=/ && m/slot=/){$fdTaken++} if (m/ fd=/ && m/closed/){ - $fdReturned++; - $simConnection--; - ($connID) = $_ =~ /conn=(\d*)\s/; - $openConnection[$connID]--; - if ($reportStats or ($verb eq "yes") || ($usage =~ /y/)) { - # if we didn't see the start time of this connection - # i.e. due to truncation or log rotation - # then just set to 0 - my $stoc = $hashes->{start_time_of_connection}->{$connID} || 0; - $hashes->{end_time_of_connection}->{$connID} = $gmtime || 0; - my $diff = $hashes->{end_time_of_connection}->{$connID} - $stoc; - $hashes->{start_time_of_connection}->{$connID} = $hashes->{end_time_of_connection}->{$connID} = 0; - if ($diff <= 1) { $latency[0]++;} - if ($diff == 2) { $latency[1]++;} - if ($diff == 3) { $latency[2]++;} - if ($diff >= 4 && $diff <=5 ) { $latency[3]++;} - if ($diff >= 6 && $diff <=10 ) { $latency[4]++;} - if ($diff >= 11 && $diff <=15 ) { $latency[5]++;} - if ($diff >= 16) { $latency[6] ++;} - } + handleConnClose($connID); } if (m/ BIND/ && $_ =~ /dn=\"(.*)\" method/i ){ my $binddn = $1; @@ -1761,13 +1800,13 @@ sub parseLineNormal $tmpp =~ tr/A-Z/a-z/; $hashes->{bindlist}->{$tmpp}++; if($usage =~ /f/ || $verb eq "yes"){ - $hashes->{bind_conn_op}->{"$conn,$op"} = $tmpp; + $hashes->{bind_conn_op}->{"$serverRestartCount,$conn,$op"} = $tmpp; } } else { $anonymousBindCount++; $hashes->{bindlist}->{"Anonymous Binds"}++; if($usage =~ /f/ || $verb eq "yes"){ - $hashes->{bind_conn_op}->{"$conn,$op"} = ""; + $hashes->{bind_conn_op}->{"$serverRestartCount,$conn,$op"} = ""; } inc_stats('anonbind',$s_stats,$m_stats); } @@ -1775,7 +1814,7 @@ sub parseLineNormal if (m/ UNBIND/){ $unbindCount++; if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{unbind_conn_op}->{"$1,$2"}++;} + if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{unbind_conn_op}->{"$serverRestartCount,$1,$2"}++;} } } if (m/ RESULT err=/ && m/ notes=[A-Z,]*P/){ @@ -1795,11 +1834,11 @@ sub parseLineNormal # unindexed search stat, as VLV unindexed searches aren't that bad $unindexedSrchCountNotesA++; if($reportStats){ inc_stats('notesA',$s_stats,$m_stats); } - if ($usage =~ /u/ || $verb eq "yes"){ - $hashes->{notesa_conn_op}->{"$con,$op"}++; - if ($_ =~ /etime= *([0-9.]+)/i ){ $hashes->{etime_conn_op}->{"$con,$op"} = $1; } - if ($_ =~ / *([0-9a-z:\/]+)/i){ $hashes->{time_conn_op}->{"$con,$op"} = $1; } - if ($_ =~ /nentries= *([0-9]+)/i ){ $hashes->{nentries_conn_op}->{"$con,$op"} = $1; } + if ($usage =~ /u/ || $usage =~ /U/ || $verb eq "yes"){ + $hashes->{notesa_conn_op}->{"$serverRestartCount,$con,$op"}++; + if ($_ =~ /etime= *([0-9.]+)/i ){ $hashes->{etime_conn_op}->{"$serverRestartCount,$con,$op"} = $1; } + if ($_ =~ / *([0-9a-z:\/]+)/i){ $hashes->{time_conn_op}->{"$serverRestartCount,$con,$op"} = $1; } + if ($_ =~ /nentries= *([0-9]+)/i ){ $hashes->{nentries_conn_op}->{"$serverRestartCount,$con,$op"} = $1; } } } $isVlvNotes = 0; @@ -1818,11 +1857,11 @@ sub parseLineNormal # unindexed search stat, as VLV unindexed searches aren't that bad $unindexedSrchCountNotesU++; if($reportStats){ inc_stats('notesU',$s_stats,$m_stats); } - if ($usage =~ /u/ || $verb eq "yes"){ - $hashes->{notesu_conn_op}->{"$con,$op"}++; - if ($_ =~ /etime= *([0-9.]+)/i ){ $hashes->{etime_conn_op}->{"$con,$op"} = $1; } - if ($_ =~ / *([0-9a-z:\/]+)/i){ $hashes->{time_conn_op}->{"$con,$op"} = $1; } - if ($_ =~ /nentries= *([0-9]+)/i ){ $hashes->{nentries_conn_op}->{"$con,$op"} = $1; } + if ($usage =~ /u/ || $usage =~ /U/ || $verb eq "yes"){ + $hashes->{notesu_conn_op}->{"$serverRestartCount,$con,$op"}++; + if ($_ =~ /etime= *([0-9.]+)/i ){ $hashes->{etime_conn_op}->{"$serverRestartCount,$con,$op"} = $1; } + if ($_ =~ / *([0-9a-z:\/]+)/i){ $hashes->{time_conn_op}->{"$serverRestartCount,$con,$op"} = $1; } + if ($_ =~ /nentries= *([0-9]+)/i ){ $hashes->{nentries_conn_op}->{"$serverRestartCount,$con,$op"} = $1; } } } $isVlvNotes = 0; @@ -1885,14 +1924,14 @@ sub parseLineNormal $hashes->{ip_hash}->{$ip}++; if ($_ =~ /conn= *([0-9A-Z]+)/i ){ if ($exc ne "yes"){ - $hashes->{conn_hash}->{$1} = $ip; + $hashes->{conn_hash}->{"$serverRestartCount,$1"} = $ip; } } } if (m/- A1/){ if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; - $ip = getIPfromConn($1); + $ip = getIPfromConn($1, $serverRestartCount); for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } @@ -1906,7 +1945,7 @@ sub parseLineNormal if (m/- B1/){ if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; - $ip = getIPfromConn($1); + $ip = getIPfromConn($1, $serverRestartCount); for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } @@ -1920,7 +1959,7 @@ sub parseLineNormal if (m/- B4/){ if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; - $ip = getIPfromConn($1); + $ip = getIPfromConn($1, $serverRestartCount); for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } @@ -1934,7 +1973,7 @@ sub parseLineNormal if (m/- T1/){ if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; - $ip = getIPfromConn($1); + $ip = getIPfromConn($1, $serverRestartCount); for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } @@ -1948,7 +1987,7 @@ sub parseLineNormal if (m/- T2/){ if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; - $ip = getIPfromConn($1); + $ip = getIPfromConn($1, $serverRestartCount); for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } @@ -1962,7 +2001,7 @@ sub parseLineNormal if (m/- B2/){ if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; - $ip = getIPfromConn($1); + $ip = getIPfromConn($1, $serverRestartCount); $maxBerSizeCount++; for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} @@ -1977,7 +2016,7 @@ sub parseLineNormal if (m/- B3/){ if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; - $ip = getIPfromConn($1); + $ip = getIPfromConn($1, $serverRestartCount); for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } @@ -1991,7 +2030,7 @@ sub parseLineNormal if (m/- R1/){ if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; - $ip = getIPfromConn($1); + $ip = getIPfromConn($1, $serverRestartCount); for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } @@ -2005,7 +2044,7 @@ sub parseLineNormal if (m/- P1/){ if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; - $ip = getIPfromConn($1); + $ip = getIPfromConn($1, $serverRestartCount); for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } @@ -2019,7 +2058,7 @@ sub parseLineNormal if (m/- P2/){ if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; - $ip = getIPfromConn($1); + $ip = getIPfromConn($1, $serverRestartCount); for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } @@ -2033,7 +2072,7 @@ sub parseLineNormal if (m/- U1/){ if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; - $ip = getIPfromConn($1); + $ip = getIPfromConn($1, $serverRestartCount); for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } @@ -2062,10 +2101,10 @@ sub parseLineNormal if ($_ =~ /oid=\" *([0-9\.]+)/i ){ $hashes->{oid}->{$1}++; } if ($1 && $1 eq $startTLSoid){$startTLSCount++;} if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{ext_conn_op}->{"$1,$2"}++;} + if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{ext_conn_op}->{"$serverRestartCount,$1,$2"}++;} } } - if (($usage =~ /l/ || $verb eq "yes") and / SRCH /){ + if (($usage =~ /l/ || $verb eq "yes" || $usage =~ /u/ || $usage =~ /U/) and / SRCH /){ my ($filterConn, $filterOp); if (/ SRCH / && / attrs=/ && $_ =~ /filter=\"(.*)\" /i ){ $tmpp = $1; @@ -2083,13 +2122,13 @@ sub parseLineNormal if ($_ =~ /op= *([0-9\-]+)/i) { $filterOp = $1; } } $filterCount++; - if($usage =~ /u/ || $verb eq "yes"){ + if($usage =~ /u/ || $usage =~ /U/ || $verb eq "yes"){ # we only need this for the unindexed search report - $hashes->{filter_conn_op}->{"$filterConn,$filterOp"} = $tmpp; + $hashes->{filter_conn_op}->{"$serverRestartCount,$filterConn,$filterOp"} = $tmpp; } } - if ($usage =~ /a/ || $verb eq "yes"){ - if (/ SRCH / && $_ =~ /base=\"(.*)\" scope/i ){ + if (($usage =~ /a/ || $verb eq "yes" || $usage =~ /u/ || $usage =~ /U/) and / SRCH /){ + if ($_ =~ /base=\"(.*)\" scope/i ){ my ($conn, $op, $scopeVal); if ($1 eq ""){ $tmpp = "Root DSE"; @@ -2110,10 +2149,10 @@ sub parseLineNormal if ($_ =~ /op= *([0-9\-]+)/i) { $op = $1; } - if($usage =~ /u/ || $verb eq "yes"){ + if($usage =~ /u/ || $usage =~ /U/ || $verb eq "yes"){ # we only need this for the unindexed search report - $hashes->{base_conn_op}->{"$conn,$op"} = $tmpp; - $hashes->{scope_conn_op}->{"$conn,$op"} = $scopeTxt[$scopeVal]; + $hashes->{base_conn_op}->{"$serverRestartCount,$conn,$op"} = $tmpp; + $hashes->{scope_conn_op}->{"$serverRestartCount,$conn,$op"} = $scopeTxt[$scopeVal]; } $baseCount++; $scopeCount++; @@ -2133,7 +2172,9 @@ sub parseLineNormal $badPwdCount++; } elsif (/ err=49 tag=/ ){ $badPwdCount++; - if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{badpwd_conn_op}->{"$1,$2"}++;} + if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ + $hashes->{badpwd_conn_op}->{"$serverRestartCount,$1,$2"} = getIPfromConn($1, $serverRestartCount); + } } } if (/ BIND / && /method=sasl/i){ @@ -2394,9 +2435,11 @@ sub getIPfromConn { my $connid = shift; - if (exists($hashes->{conn_hash}->{$connid}) && - defined($hashes->{conn_hash}->{$connid})) { - return $hashes->{conn_hash}->{$connid}; + my $serverRestartCount = shift; + my $key = "$serverRestartCount,$connid"; + if (exists($hashes->{conn_hash}->{$key}) && + defined($hashes->{conn_hash}->{$key})) { + return $hashes->{conn_hash}->{$key}; } return "Unknown_Host"; diff --git a/man/man1/logconv.pl.1 b/man/man1/logconv.pl.1 index c2a8cd8..46e7c96 100644 --- a/man/man1/logconv.pl.1 +++ b/man/man1/logconv.pl.1 @@ -21,7 +21,7 @@ logconv.pl \- analyzes Directory Server access log files .B logconv.pl [\fI\-h\fR] [\fI\-d <rootDN>\fR] [\fI\-s <size limit>\fR] [\fI\-v\fR] [\fI\-V\fR] [\fI\-S <start time>\fR] [\fI\-E <end time>\fR] -[\fI\-efcibaltnxgju\fR] [\fI access log ... ... \fR] +[\fI\-efcibaltnxgjuU\fR] [\fI access log ... ... \fR] .PP .SH DESCRIPTION Analyzes Directory Server access log files for specific information defined on the command @@ -68,10 +68,10 @@ This option creates a CSV report for spreadsheets. .B \fB\-B, \-\-bind\fR <ALL | ANONYMOUS | "Bind DN"> This generates a report based on either ALL bind dn's, anonymous binds, or a specific DN. .TP -\fB\-V, \-\-verbose\fR <enable verbose output \- includes all stats listed below> +\fB\-V, \-\-verbose\fR <enable verbose output \- includes all stats listed below except U> Verbose output .TP -.B \fB\-[efcibaltnxgju]\fR +.B \fB\-[efcibaltnxgjuU]\fR .br \fBe\fR Error Code stats .br @@ -99,11 +99,13 @@ Verbose output .br \fBj\fR Recommendations .br -\fBu\fR Unindexed Search Stats +\fBu\fR Unindexed Search Stats (very detailed) .br \fBy\fR Connection Latency Stats .br \fBp\fR Open Connection ID Stats +.br +\fBU\fR Unindexed Search Summary .PP .SH USAGE Examples: commit 223c0535e09b46b3024dc9977eee048f824f2b24 Author: Rich Megginson <rmeggins(a)redhat.com&gt; Date: Tue Jul 23 17:05:32 2013 -0600 Ticket #47501 logconv.pl uses /var/tmp for BDB temp files https://fedorahosted.org/389/ticket/47501 Reviewed by: mreynolds (Thanks!) Branch: master Fix Description: Tied arrays with DB_RECNO require a backing text file. By default this goes in /var/tmp. This can be controlled with RECNOINFO bfname. However, the use of arrays was killing performance. This patch replaces all arrays with tied hashes. In addition, this patch fixes the notes=U handling to also handle notes=U,P and notes=A,P Platforms tested: RHEL6 x86_64 Flag Day: no Doc impact: no (cherry picked from commit 4d20922423c9e22c858e0be7dd317591631cf357) (cherry picked from commit d890e65b4ab15d0a836dbb248da8606867938630) diff --git a/ldap/admin/src/logconv.pl b/ldap/admin/src/logconv.pl index 3a6fa01..ca07a3a 100755 --- a/ldap/admin/src/logconv.pl +++ b/ldap/admin/src/logconv.pl @@ -67,7 +67,7 @@ if ($#ARGV < 0){; my $file_count = 0; my $arg_count = 0; -my $logversion = "7.0"; +my $logversion = "8.0"; my $sizeCount = "20"; my $startFlag = 0; my $startTime = 0; @@ -76,6 +76,8 @@ my $endTime = 0; my $reportStats = ""; my $dataLocation = "/tmp"; my $startTLSoid = "1.3.6.1.4.1.1466.20037"; +my @statnames=qw(last last_str results srch add mod modrdn moddn cmp del abandon + conns sslconns bind anonbind unbind notesA notesU etime); my $s_stats = new_stats_block( ); my $m_stats = new_stats_block( ); my $verb = "no"; @@ -95,8 +97,6 @@ my %connList; my %bindReport; my @vlvconn; my @vlvop; -my @start_time_of_connection; -my @end_time_of_connection; my @fds; my $fdds = 0; my $reportBinds = "no"; @@ -179,7 +179,7 @@ if ($sizeCount eq "all"){$sizeCount = "100000";} ####################################### # # -# Initialize Arrays and variables # +# Initialize Hashes and variables # # # ####################################### @@ -255,21 +255,17 @@ map {$conn{$_} = $_} @conncodes; # hash db-backed hashes my @hashnames = qw(attr rc src rsrc excount conn_hash ip_hash conncount nentries - filter base ds6xbadpwd saslmech bindlist etime oid); + filter base ds6xbadpwd saslmech bindlist etime oid + start_time_of_connection end_time_of_connection + notesa_conn_op notesu_conn_op etime_conn_op nentries_conn_op + optype_conn_op time_conn_op srch_conn_op del_conn_op mod_conn_op + mdn_conn_op cmp_conn_op bind_conn_op unbind_conn_op ext_conn_op + abandon_conn_op badpwd_conn_op); # need per connection code ip address counts - so use a hash table # for each connection code - key is ip, val is count push @hashnames, @conncodes; my $hashes = openHashFiles($dataLocation, @hashnames); -# recno db-backed arrays/lists -my @arraynames = qw(srchconn srchop delconn delop modconn modop addconn addop modrdnconn modrdnop - cmpconn cmpop targetconn targetop msgid bindconn bindop binddn unbindconn unbindop - extconn extop notesAetime notesAconn notesAop notesAtime notesAnentries - notesUetime notesUconn notesUop notesUtime notesUnentries badpwdconn - badpwdop badpwdip baseval baseconn baseop scopeval scopeconn scopeop - filterval filterconn filterop); -my $arrays = openArrayFiles($dataLocation, @arraynames); - $needCleanup = 1; my @err; @@ -636,6 +632,12 @@ Binds: @<<<<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< . write STDOUT; +# format/write messes up emacs autoformatting - inserting this dummy function +# fixes that +sub dummy { + my $bar = shift; +} + print "\n"; print "Proxied Auth Operations: $proxiedAuthCount\n"; print "Persistent Searches: $persistentSrchCount\n"; @@ -658,115 +660,75 @@ print "Unindexed Components: $unindexedSrchCountNotesU\n"; if ($verb eq "yes" || $usage =~ /u/){ if ($unindexedSrchCountNotesA > 0){ my $conn_hash = $hashes->{conn_hash}; - my $notesConn = $arrays->{notesAconn}; - my $notesOp = $arrays->{notesAop}; - my $notesEtime = $arrays->{notesAetime}; - my $notesTime = $arrays->{notesAtime}; - my $notesNentries = $arrays->{notesAnentries}; - my $base_val = $arrays->{baseval}; - my $base_conn = $arrays->{baseconn}; - my $base_op = $arrays->{baseop}; - my $scope_val = $arrays->{scopeval}; - my $scope_conn = $arrays->{scopeconn}; - my $scope_op = $arrays->{scopeop}; - my $filter_val = $arrays->{filterval}; - my $filter_conn = $arrays->{filterconn}; - my $filter_op = $arrays->{filterop}; - my $notesCount = "1"; + my $notesa_conn_op = $hashes->{notesa_conn_op}; + my $time_conn_op = $hashes->{time_conn_op}; + my $etime_conn_op = $hashes->{etime_conn_op}; + my $nentries_conn_op = $hashes->{nentries_conn_op}; + my $base_conn_op = $hashes->{base_conn_op}; + my $scope_conn_op = $hashes->{scope_conn_op}; + my $filter_conn_op = $hashes->{filter_conn_op}; + + my $notesCount = 1; my $unindexedIp; - - for (my $n = 0; $n < scalar(@{$notesEtime}); $n++){ - if(!$notesConn->[$n] || $notesConn->[$n] eq "" || - !$conn_hash->{$notesConn->[$n]} || $conn_hash->{$notesConn->[$n]} eq ""){ - $unindexedIp = "?"; + while (my ($conn_op, $count) = each %{$notesa_conn_op}) { + my ($conn, $op) = split(",", $conn_op); + if (exists($conn_hash->{$conn}) && defined($conn_hash->{$conn})) { + $unindexedIp = $conn_hash->{$conn}; } else { - $unindexedIp = $conn_hash->{$notesConn->[$n]}; + $unindexedIp = "?"; } - print "\n Unindexed Search #".$notesCount."\n"; $notesCount++; - print " - Date/Time: $notesTime->[$n]\n"; - print " - Connection Number: $notesConn->[$n]\n"; - print " - Operation Number: $notesOp->[$n]\n"; - print " - Etime: $notesEtime->[$n]\n"; - print " - Nentries: $notesNentries->[$n]\n"; + print "\n Unindexed Components #".$notesCount."\n"; $notesCount++; + print " - Date/Time: $time_conn_op->{$conn_op}\n"; + print " - Connection Number: $conn\n"; + print " - Operation Number: $op\n"; + print " - Etime: $etime_conn_op->{$conn_op}\n"; + print " - Nentries: $nentries_conn_op->{$conn_op}\n"; print " - IP Address: $unindexedIp\n"; - - for (my $nnn = 0; $nnn < $baseCount; $nnn++){ - if(!$base_conn->[$nnn] || !$base_op->[$nnn]){ next; } - if ($notesConn->[$n] eq $base_conn->[$nnn] && $notesOp->[$n] eq $base_op->[$nnn]){ - print " - Search Base: $base_val->[$nnn]\n"; - last; - } + if (exists($base_conn_op->{$conn_op}) && defined($base_conn_op->{$conn_op})) { + print " - Search Base: $base_conn_op->{$conn_op}\n"; } - for (my $nnn = 0; $nnn < $scopeCount; $nnn++){ - if(!$scope_conn->[$nnn] || !$scope_op->[$nnn]){ next; } - if ($notesConn->[$n] eq $scope_conn->[$nnn] && $notesOp->[$n] eq $scope_op->[$nnn]){ - print " - Search Scope: $scope_val->[$nnn]\n"; - last; - } + if (exists($scope_conn_op->{$conn_op}) && defined($scope_conn_op->{$conn_op})) { + print " - Search Scope: $scope_conn_op->{$conn_op}\n"; } - for (my $nnn = 0; $nnn < $filterCount; $nnn++){ - if(!$filter_conn->[$nnn] || !$filter_op->[$nnn]){ next; } - if ($notesConn->[$n] eq $filter_conn->[$nnn] && $notesOp->[$n] eq $filter_op->[$nnn]){ - print " - Search Filter: $filter_val->[$nnn]\n"; - last; - } + if (exists($filter_conn_op->{$conn_op}) && defined($filter_conn_op->{$conn_op})) { + print " - Search Filter: $filter_conn_op->{$conn_op}\n"; } } } if ($unindexedSrchCountNotesU > 0){ my $conn_hash = $hashes->{conn_hash}; - my $notesConn = $arrays->{notesUconn}; - my $notesOp = $arrays->{notesUop}; - my $notesEtime = $arrays->{notesUetime}; - my $notesTime = $arrays->{notesUtime}; - my $notesNentries = $arrays->{notesUnentries}; - my $base_val = $arrays->{baseval}; - my $base_conn = $arrays->{baseconn}; - my $base_op = $arrays->{baseop}; - my $scope_val = $arrays->{scopeval}; - my $scope_conn = $arrays->{scopeconn}; - my $scope_op = $arrays->{scopeop}; - my $filter_val = $arrays->{filterval}; - my $filter_conn = $arrays->{filterconn}; - my $filter_op = $arrays->{filterop}; - - my $notesCount = "1"; + my $notesu_conn_op = $hashes->{notesu_conn_op}; + my $time_conn_op = $hashes->{time_conn_op}; + my $etime_conn_op = $hashes->{etime_conn_op}; + my $nentries_conn_op = $hashes->{nentries_conn_op}; + my $base_conn_op = $hashes->{base_conn_op}; + my $scope_conn_op = $hashes->{scope_conn_op}; + my $filter_conn_op = $hashes->{filter_conn_op}; + + my $notesCount = 1; my $unindexedIp; - for (my $n = 0; $n < scalar(@{$notesEtime}); $n++){ - if(!$notesConn->[$n] || $notesConn->[$n] eq "" || - !$conn_hash->{$notesConn->[$n]} || $conn_hash->{$notesConn->[$n]} eq ""){ - $unindexedIp = "?"; + while (my ($conn_op, $count) = each %{$notesu_conn_op}) { + my ($conn, $op) = split(",", $conn_op); + if (exists($conn_hash->{$conn}) && defined($conn_hash->{$conn})) { + $unindexedIp = $conn_hash->{$conn}; } else { - $unindexedIp = $conn_hash->{$notesConn->[$n]}; + $unindexedIp = "?"; } print "\n Unindexed Components #".$notesCount."\n"; $notesCount++; - print " - Date/Time: $notesTime->[$n]\n"; - print " - Connection Number: $notesConn->[$n]\n"; - print " - Operation Number: $notesOp->[$n]\n"; - print " - Etime: $notesEtime->[$n]\n"; - print " - Nentries: $notesNentries->[$n]\n"; + print " - Date/Time: $time_conn_op->{$conn_op}\n"; + print " - Connection Number: $conn\n"; + print " - Operation Number: $op\n"; + print " - Etime: $etime_conn_op->{$conn_op}\n"; + print " - Nentries: $nentries_conn_op->{$conn_op}\n"; print " - IP Address: $unindexedIp\n"; - - for (my $nnn = 0; $nnn < $baseCount; $nnn++){ - if(!$base_conn->[$nnn] || !$base_op->[$nnn]){ next; } - if ($notesConn->[$n] eq $base_conn->[$nnn] && $notesOp->[$n] eq $base_op->[$nnn]){ - print " - Search Base: $base_val->[$nnn]\n"; - last; - } + if (exists($base_conn_op->{$conn_op}) && defined($base_conn_op->{$conn_op})) { + print " - Search Base: $base_conn_op->{$conn_op}\n"; } - for (my $nnn = 0; $nnn < $scopeCount; $nnn++){ - if(!$scope_conn->[$nnn] || !$scope_op->[$nnn]){ next; } - if ($notesConn->[$n] eq $scope_conn->[$nnn] && $notesOp->[$n] eq $scope_op->[$nnn]){ - print " - Search Scope: $scope_val->[$nnn]\n"; - last; - } + if (exists($scope_conn_op->{$conn_op}) && defined($scope_conn_op->{$conn_op})) { + print " - Search Scope: $scope_conn_op->{$conn_op}\n"; } - for (my $nnn = 0; $nnn < $filterCount; $nnn++){ - if(!$filter_conn->[$nnn] || !$filter_op->[$nnn]){ next; } - if ($notesConn->[$n] eq $filter_conn->[$nnn] && $notesOp->[$n] eq $filter_op->[$nnn]){ - print " - Search Filter: $filter_val->[$nnn]\n"; - last; - } + if (exists($filter_conn_op->{$conn_op}) && defined($filter_conn_op->{$conn_op})) { + print " - Search Filter: $filter_conn_op->{$conn_op}\n"; } } } @@ -800,7 +762,7 @@ if ($connResetByPeerCount > 0){ push @retext, sprintf " - %-4s (%2s) %-40s\n",$src->{$key},$conn{$key},$connmsg{$key}; } } - print @retext; + print @retext; print "\n"; } @@ -813,8 +775,8 @@ if ($resourceUnavailCount > 0){ if ($conn{$key} eq ""){$conn{$key} = "**Resource Issue**";} push @rtext, sprintf " - %-4s (%2s) %-40s\n",$rsrc->{$key},$conn{$key},$connmsg{$key}; } - } - print @rtext; + } + print @rtext; } print "Max BER Size Exceeded: $maxBerSizeCount\n"; print "\n"; @@ -829,7 +791,7 @@ print " - SASL Binds: $saslBindCount\n"; if ($saslBindCount > 0){ my $saslmech = $hashes->{saslmech}; foreach my $saslb ( sort {$saslmech->{$b} <=> $saslmech->{$a} } (keys %{$saslmech}) ){ - printf " %-4s %-12s\n",$saslmech->{$saslb}, $saslb; + printf " %-4s %-12s\n",$saslmech->{$saslb}, $saslb; } } @@ -914,13 +876,13 @@ if ($usage =~ /e/i || $verb eq "yes"){ for (my $i = 0; $i <= $#errtext; $i++){ $errtext[$i] =~ s/\n//g; print "\n" . $errtext[$i]; - } + } } #################################### -# # +# # # Print Failed Logins # -# # +# # #################################### if ($verb eq "yes" || $usage =~ /f/ ){ @@ -936,19 +898,20 @@ if ($verb eq "yes" || $usage =~ /f/ ){ $ds6loop++; } } else { - my $bindVal = $arrays->{binddn}; - my $bindConn = $arrays->{bindconn}; - my $bindOp = $arrays->{bindop}; - my $badPasswordConn = $arrays->{badpwdconn}; - my $badPasswordOp = $arrays->{badpwdop}; - my $badPasswordIp = $arrays->{badpwdip}; + # key is conn,op - val is binddn + my $bind_conn_op = $hashes->{bind_conn_op}; + # key is conn,op - val is count + my $badpwd_conn_op = $hashes->{badpwd_conn_op}; + # key is binddn - val is count my %badPassword = (); - for (my $ii =0 ; $ii < $badPwdCount; $ii++){ - for (my $i = 0; $i < $bindCount; $i++){ - if ($badPasswordConn->[$ii] eq $bindConn->[$i] && $badPasswordOp->[$ii] eq $bindOp->[$i] ){ - $badPassword{ $bindVal->[$i] }++; - } - } + my @badPasswordIp = (); + while (my ($conn_op, $count) = each %{$badpwd_conn_op}) { + my ($conn, $op) = split(",", $conn_op); + if (exists($bind_conn_op->{$conn_op}) && defined($bind_conn_op->{$conn_op})) { + my $binddn = $bind_conn_op->{$conn_op}; + $badPassword{$binddn}++; + push @badPasswordIp, getIPfromConn($conn); + } } # sort the new hash of $badPassword{} my $bpTotal = 0; @@ -961,7 +924,7 @@ if ($verb eq "yes" || $usage =~ /f/ ){ } print "\nFrom the IP address(s) :\n\n"; for (my $i=0; $i<$badPwdCount; $i++) { - print "\t\t$badPasswordIp->[$i]\n"; + print "\t\t$badPasswordIp[$i]\n"; } if ($bpTotal > $badPwdCount){ print "\n** Warning : Wrongly reported failed login attempts : ". ($bpTotal - $badPwdCount) . "\n"; @@ -982,11 +945,11 @@ if ($connCodeCount > 0){ print "\n\n----- Total Connection Codes -----\n\n"; my $conncount = $hashes->{conncount}; my @conntext; - foreach my $key (sort { $conncount->{$b} <=> $conncount->{$a} } keys %{$conncount}) { - if ($conncount->{$key} > 0){ + foreach my $key (sort { $conncount->{$b} <=> $conncount->{$a} } keys %{$conncount}) { + if ($conncount->{$key} > 0){ push @conntext, sprintf "%-4s %6s %-40s\n",$key,$conncount->{$key},$connmsg{ $key }; - } - } + } + } print @conntext; } } @@ -1005,8 +968,8 @@ if ($usage =~ /i/i || $verb eq "yes"){ my $ip_count = ($#ipkeys + 1)-($#exxCount + 1); my $ccount = 0; if ($ip_count > 0){ - print "\n\n----- Top $sizeCount Clients -----\n\n"; - print "Number of Clients: $ip_count\n\n"; + print "\n\n----- Top $sizeCount Clients -----\n\n"; + print "Number of Clients: $ip_count\n\n"; foreach my $key (sort { $ip_hash->{$b} <=> $ip_hash->{$a} } @ipkeys) { my $exc = "no"; if ($ccount > $sizeCount){ last;} @@ -1021,7 +984,7 @@ if ($usage =~ /i/i || $verb eq "yes"){ my %counts; map { $counts{$_} = $hashes->{$_}->{$key} if (defined($hashes->{$_}->{$key})) } @conncodes; foreach my $code (sort { $counts{$b} <=> $counts{$a} } keys %counts) { - if ($code eq 'count' ) { next; } + if ($code eq 'count' ) { next; } printf "%10s - %s (%s)\n", $counts{ $code }, $code, $connmsg{ $code }; } print "\n"; @@ -1042,14 +1005,14 @@ if ($usage =~ /b/i || $verb eq "yes"){ my $bind_count = $#bindkeys + 1; if ($bind_count > 0){ print "\n\n----- Top $sizeCount Bind DN's -----\n\n"; - print "Number of Unique Bind DN's: $bind_count\n\n"; + print "Number of Unique Bind DN's: $bind_count\n\n"; my $bindcount = 0; foreach my $dn (sort { $bindlist->{$b} <=> $bindlist->{$a} } @bindkeys) { - if ($bindcount < $sizeCount){ + if ($bindcount < $sizeCount){ printf "%-8s %-40s\n", $bindlist->{ $dn },$dn; } else { - last; - } + last; + } $bindcount++; } } @@ -1070,16 +1033,16 @@ if ($usage =~ /a/i || $verb eq "yes"){ print "Number of Unique Search Bases: $base_count\n\n"; my $basecount = 0; foreach my $bas (sort { $base->{$b} <=> $base->{$a} } @basekeys) { - if ($basecount < $sizeCount){ - printf "%-8s %-40s\n", $base->{ $bas },$bas; - } else { - last; - } - $basecount++; + if ($basecount < $sizeCount){ + printf "%-8s %-40s\n", $base->{ $bas },$bas; + } else { + last; + } + $basecount++; } } } - + ######################################### # # # Gather and process search filters # @@ -1091,15 +1054,15 @@ if ($usage =~ /l/ || $verb eq "yes"){ my @filterkeys = keys %{$filter}; my $filter_count = $#filterkeys + 1; if ($filter_count > 0){ - print "\n\n----- Top $sizeCount Search Filters -----\n"; + print "\n\n----- Top $sizeCount Search Filters -----\n"; print "\nNumber of Unique Search Filters: $filter_count\n\n"; my $filtercount = 0; foreach my $filt (sort { $filter->{$b} <=> $filter->{$a} } @filterkeys){ if ($filtercount < $sizeCount){ printf "%-8s %-40s\n", $filter->{$filt}, $filt; } else { - last; - } + last; + } $filtercount++; } } @@ -1108,7 +1071,7 @@ if ($usage =~ /l/ || $verb eq "yes"){ ######################################### # # # Gather and Process the unique etimes # -# # +# # ######################################### my $first; @@ -1139,7 +1102,7 @@ if ($usage =~ /t/i || $verb eq "yes"){ if ($eloop == $sizeCount) { last; } printf "%-12s %-10s\n","etime=$et",$etime->{ $et }; $eloop++; - } + } } ####################################### @@ -1156,7 +1119,7 @@ if ($usage =~ /n/i || $verb eq "yes"){ my $eloop = 0; foreach my $nentry (sort { $b <=> $a } @nkeys){ if ($eloop == $sizeCount) { last; } - printf "%-18s %12s\n","nentries=$nentry", $nentries->{ $nentry }; + printf "%-18s %12s\n","nentries=$nentry", $nentries->{ $nentry }; $eloop++; } print "\n\n----- Top $sizeCount Most returned nentries -----\n\n"; @@ -1216,7 +1179,7 @@ if ($usage =~ /x/i || $verb eq "yes"){ } ############################################ -# # +# # # Print most commonly requested attributes # # # ############################################ @@ -1235,7 +1198,7 @@ if ($usage =~ /r/i || $verb eq "yes"){ } ############################# -# # +# # # abandoned operation stats # # # ############################# @@ -1244,66 +1207,49 @@ if ($usage =~ /g/i || $verb eq "yes"){ my $abandonTotal = $srchCount + $delCount + $modCount + $addCount + $modrdnCount + $bindCount + $extopCount + $cmpCount; if ($verb eq "yes" && $abandonCount > 0 && $abandonTotal > 0){ my $conn_hash = $hashes->{conn_hash}; + my $abandon_conn_op = $hashes->{abandon_conn_op}; + my $srch_conn_op = $hashes->{srch_conn_op}; + my $del_conn_op = $hashes->{del_conn_op}; + my $add_conn_op = $hashes->{add_conn_op}; + my $mod_conn_op = $hashes->{mod_conn_op}; + my $cmp_conn_op = $hashes->{cmp_conn_op}; + my $mdn_conn_op = $hashes->{mdn_conn_op}; + my $bind_conn_op = $hashes->{bind_conn_op}; + my $unbind_conn_op = $hashes->{unbind_conn_op}; + my $ext_conn_op = $hashes->{ext_conn_op}; print "\n\n----- Abandon Request Stats -----\n\n"; - for (my $g = 0; $g < $abandonCount; $g++){ - my $conn = $arrays->{targetconn}->[$g]; - my $op = $arrays->{targetop}->[$g]; - my $msgid = $arrays->{msgid}->[$g]; - for (my $sc = 0; $sc < $srchCount; $sc++){ - if (($arrays->{srchconn}->[$sc] && $arrays->{srchop}->[$sc]) && - ($arrays->{srchconn}->[$sc] eq $conn && $arrays->{srchop}->[$sc] eq $op )){ - print " - SRCH conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; - } - } - for (my $dc = 0; $dc < $delCount; $dc++){ - if (($arrays->{delconn}->[$dc] && $arrays->{delop}->[$dc]) && - ($arrays->{delconn}->[$dc] eq $conn && $arrays->{delop}->[$dc] eq $op)){ - print " - DEL conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; - } - } - for (my $adc = 0; $adc < $addCount; $adc++){ - if (($arrays->{addconn}->[$adc] && $arrays->{addop}->[$adc]) && - ($arrays->{addconn}->[$adc] eq $conn && $arrays->{addop}->[$adc] eq $op)){ - print " - ADD conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; - } - } - for (my $mc = 0; $mc < $modCount; $mc++){ - if (($arrays->{modconn}->[$mc] && $arrays->{modop}->[$mc]) && - ($arrays->{modconn}->[$mc] eq $conn && $arrays->{modop}->[$mc] eq $op)){ - print " - MOD conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; - } - } - for (my $cc = 0; $cc < $cmpCount; $cc++){ - if (($arrays->{cmpconn}->[$cc] && $arrays->{cmpop}->[$cc]) && - ($arrays->{cmpconn}->[$cc] eq $conn && $arrays->{cmpop}->[$cc] eq $op)){ - print " - CMP conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; - } - } - for (my $mdc = 0; $mdc < $modrdnCount; $mdc++){ - if (($arrays->{modrdnconn}->[$mdc] && $arrays->{modrdnop}->[$mdc]) && - ($arrays->{modrdnconn}->[$mdc] eq $conn && $arrays->{modrdnop}->[$mdc] eq $op)){ - print " - MODRDN conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; - } - } - for (my $bcb = 0; $bcb < $bindCount; $bcb++){ - if (($arrays->{bindconn}->[$bcb] && $arrays->{bindop}->[$bcb]) && - ($arrays->{bindconn}->[$bcb] eq $conn && $arrays->{bindop}->[$bcb] eq $op)){ - print " - BIND conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; - } - } - for (my $ubc = 0; $ubc < $unbindCount; $ubc++){ - if (($arrays->{unbindconn}->[$ubc] && $arrays->{unbindop}->[$ubc]) && - ($arrays->{unbindconn}->[$ubc] eq $conn && $arrays->{unbindop}->[$ubc] eq $op)){ - print " - UNBIND conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; - } + while (my ($conn_op, $targ_msgid) = each %{$abandon_conn_op}) { + my ($conn, $op) = split(",", $conn_op); + my ($targetop, $msgid) = split(",", $targ_msgid); + my $conn_targ = "$conn,$targetop"; + my $clientIP; + if (exists($conn_hash->{$conn}) && defined($conn_hash->{$conn})) { + $clientIP = $conn_hash->{$conn}; + } else { + $clientIP = "Unknown"; } - for (my $ec = 0; $ec < $extopCount; $ec++){ - if (($arrays->{extconn}->[$ec] && $arrays->{extop}->[$ec]) && - ($arrays->{extconn}->[$ec] eq $conn && $arrays->{extop}->[$ec] eq $op)){ - print " - EXT conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; - } + if (exists($srch_conn_op->{$conn_targ}) && defined($srch_conn_op->{$conn_targ})) { + print " - SRCH conn=$conn op=$targetop msgid=$msgid client=$clientIP\n"; + } elsif (exists($del_conn_op->{$conn_targ}) && defined($del_conn_op->{$conn_targ})) { + print " - DEL conn=$conn op=$targetop msgid=$msgid client=$clientIP\n"; + } elsif (exists($add_conn_op->{$conn_targ}) && defined($add_conn_op->{$conn_targ})) { + print " - ADD conn=$conn op=$targetop msgid=$msgid client=$clientIP\n"; + } elsif (exists($mod_conn_op->{$conn_targ}) && defined($mod_conn_op->{$conn_targ})) { + print " - MOD conn=$conn op=$targetop msgid=$msgid client=$clientIP\n"; + } elsif (exists($cmp_conn_op->{$conn_targ}) && defined($cmp_conn_op->{$conn_targ})) { + print " - CMP conn=$conn op=$targetop msgid=$msgid client=$clientIP\n"; + } elsif (exists($mdn_conn_op->{$conn_targ}) && defined($mdn_conn_op->{$conn_targ})) { + print " - MODRDN conn=$conn op=$targetop msgid=$msgid client=$clientIP\n"; + } elsif (exists($bind_conn_op->{$conn_targ}) && defined($bind_conn_op->{$conn_targ})) { + print " - BIND conn=$conn op=$targetop msgid=$msgid client=$clientIP\n"; + } elsif (exists($unbind_conn_op->{$conn_targ}) && defined($unbind_conn_op->{$conn_targ})) { + print " - UNBIND conn=$conn op=$targetop msgid=$msgid client=$clientIP\n"; + } elsif (exists($ext_conn_op->{$conn_targ}) && defined($ext_conn_op->{$conn_targ})) { + print " - EXT conn=$conn op=$targetop msgid=$msgid client=$clientIP\n"; + } else { + print " - UNKNOWN conn=$conn op=$targetop msgid=$msgid client=$clientIP\n"; } } } @@ -1390,8 +1336,8 @@ sub displayUsage { print "Usage:\n\n"; print " ./logconv.pl [-h] [-d|--rootdn <rootDN>] [-s|--sizeLimit <size limit>] [-v|verison] [-Vi|verbose]\n"; - print " [-S|--startTime <start time>] [-E|--endTime <end time>] \n"; - print " [-efcibaltnxrgjuyp] [ access log ... ... ]\n\n"; + print " [-S|--startTime <start time>] [-E|--endTime <end time>] \n"; + print " [-efcibaltnxrgjuyp] [ access log ... ... ]\n\n"; print "- Commandline Switches:\n\n"; @@ -1400,7 +1346,7 @@ sub displayUsage { print " -D, --data <Location for temporary data files> default is \"/tmp\"\n"; print " -s, --sizeLimit <Number of results to return per catagory> default is 20\n"; print " -X, --excludeIP <IP address to exclude from connection stats> E.g. Load balancers\n"; - print " -v, --version show version of tool\n"; + print " -v, --version show version of tool\n"; print " -S, --startTime <time to begin analyzing logfile from>\n"; print " E.g. \"[28/Mar/2002:13:14:22 -0800]\"\n"; print " -E, --endTime <time to stop analyzing logfile>\n"; @@ -1450,7 +1396,7 @@ sub displayUsage { sub parseLine { if($reportBinds eq "yes"){ - &parseLineBind(); + &parseLineBind(); } else { &parseLineNormal(); } @@ -1469,26 +1415,26 @@ parseLineBind { return if $_ =~ /^\s/; if($firstFile == 1 && $_ =~ /^\[/){ - $start = $_; - if ($start =~ / *([0-9a-z:\/]+)/i){$start=$1;} - $firstFile = 0; + $start = $_; + if ($start =~ / *([0-9a-z:\/]+)/i){$start=$1;} + $firstFile = 0; } if ($endFlag != 1 && $_ =~ /^\[/ && $_ =~ / *([0-9a-z:\/]+)/i){ $end =$1; } if ($startTime && !$startFlag) { - if (index($_, $startTime) == 0) { - $startFlag = 1; - ($start) = $startTime =~ /\D*(\S*)/; - } else { - return; - } + if (index($_, $startTime) == 0) { + $startFlag = 1; + ($start) = $startTime =~ /\D*(\S*)/; + } else { + return; + } } if ($endTime && !$endFlag) { - if (index($_, $endTime) == 0) { - $endFlag = 1; - ($end) = $endTime =~ /\D*(\S*)/; - } + if (index($_, $endTime) == 0) { + $endFlag = 1; + ($end) = $endTime =~ /\D*(\S*)/; + } } if ($_ =~ /connection from *([0-9A-Fa-f\.\:]+)/i ) { my $skip = "yes"; @@ -1497,8 +1443,8 @@ parseLineBind { $skip = "yes"; last; } - } - if ($skip eq "yes"){ + } + if ($skip eq "yes"){ return ; } $ip = $1; @@ -1507,9 +1453,9 @@ parseLineBind { } return; } - if (/ BIND/ && $_ =~ /dn=\"(.*)\" method/i ){ + if (/ BIND/ && $_ =~ /dn=\"(.*)\" method/i ){ my $dn; - if ($1 eq ""){ + if ($1 eq ""){ $dn = "Anonymous"; } else { $dn = $1; @@ -1522,7 +1468,7 @@ parseLineBind { } } $bindReport{$dn}{"binds"}++; - if ($bindReport{$dn}{"binds"} == 1){ + if ($bindReport{$dn}{"binds"} == 1){ # For hashes we need to init the counters $bindReport{$dn}{"srch"} = 0; $bindReport{$dn}{"add"} = 0; @@ -1537,7 +1483,7 @@ parseLineBind { $bindReport{$dn}{"conn"} = $bindReport{$dn}{"conn"} . " $1 "; } return; - } + } if (/ RESULT err=49 /){ processOpForBindReport("failedBind",$logline); } @@ -1547,7 +1493,7 @@ parseLineBind { processOpForBindReport("add",$logline); } elsif (/ MOD dn=/){ processOpForBindReport("mod",$logline); - } elsif (/ DEL dn=/){ + } elsif (/ DEL dn=/){ processOpForBindReport("del",$logline); } elsif (/ MODRDN dn=/){ processOpForBindReport("modrdn",$logline); @@ -1555,7 +1501,7 @@ parseLineBind { processOpForBindReport("cmp",$logline); } elsif (/ EXT oid=/){ processOpForBindReport("ext",$logline); - } + } } sub @@ -1625,13 +1571,13 @@ sub parseLineNormal if(!defined($lastzone) or $tzone ne $lastzone) { - # tz offset change - $lastzone=$tzone; - my ($sign,$hr,$min) = $tzone =~ m/(.)(\d\d)(\d\d)/; - $tzoff = $hr*3600 + $min*60; - $tzoff *= -1 - if $sign eq '-'; - # to be subtracted from converted values. + # tz offset change + $lastzone=$tzone; + my ($sign,$hr,$min) = $tzone =~ m/(.)(\d\d)(\d\d)/; + $tzoff = $hr*3600 + $min*60; + $tzoff *= -1 + if $sign eq '-'; + # to be subtracted from converted values. } my ($date, $hr, $min, $sec) = split (':', $time); my ($day, $mon, $yr) = split ('/', $date); @@ -1641,15 +1587,15 @@ sub parseLineNormal reset_stats_block( $s_stats, $gmtime, $time.' '.$tzone ); if (!defined($last_min) or $newmin != $last_min) { - print_stats_block( $m_stats ); - $time =~ s/\d\d$/00/; - reset_stats_block( $m_stats, $newmin, $time.' '.$tzone ); - $last_min = $newmin; + print_stats_block( $m_stats ); + $time =~ s/\d\d$/00/; + reset_stats_block( $m_stats, $newmin, $time.' '.$tzone ); + $last_min = $newmin; } } - if (m/ RESULT err/){ - $allResults++; + if (m/ RESULT err/){ + $allResults++; if($reportStats){ inc_stats('results',$s_stats,$m_stats); } } if (m/ SRCH/){ @@ -1666,58 +1612,57 @@ sub parseLineNormal $anyAttrs++; } if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{srchconn}}, $1;} - if ($_ =~ /op= *([0-9\-]+)/i){ push @{$arrays->{srchop}}, $1;} + if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{srch_conn_op}->{"$1,$2"}++;} } } if (m/ DEL/){ $delCount++; if($reportStats){ inc_stats('del',$s_stats,$m_stats); } if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{delconn}}, $1;} - if ($_ =~ /op= *([0-9\-]+)/i){ push @{$arrays->{delop}}, $1;} + if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{del_conn_op}->{"$1,$2"}++;} } } if (m/ MOD dn=/){ $modCount++; if($reportStats){ inc_stats('mod',$s_stats,$m_stats); } if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{modconn}}, $1;} - if ($_ =~ /op= *([0-9\-]+)/i){ push @{$arrays->{modop}}, $1; } + if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{mod_conn_op}->{"$1,$2"}++;} } } if (m/ ADD/){ $addCount++; if($reportStats){ inc_stats('add',$s_stats,$m_stats); } if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{addconn}}, $1; } - if ($_ =~ /op= *([0-9\-]+)/i){ push @{$arrays->{addop}}, $1; } + if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{add_conn_op}->{"$1,$2"}++;} } } if (m/ MODRDN/){ $modrdnCount++; if($reportStats){ inc_stats('modrdn',$s_stats,$m_stats); } if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{modrdnconn}}, $1; } - if ($_ =~ /op= *([0-9\-]+)/i){ push @{$arrays->{modrdnop}}, $1; } + if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{mdn_conn_op}->{"$1,$2"}++;} } } if (m/ CMP dn=/){ $cmpCount++; if($reportStats){ inc_stats('cmp',$s_stats,$m_stats); } if ($verb eq "yes" || $usage =~ /g/i){ - if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{cmpconn}}, $1;} - if ($_ =~ /op= *([0-9\-]+)/i){ push @{$arrays->{cmpop}}, $1;} + if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{cmp_conn_op}->{"$1,$2"}++;} } } if (m/ ABANDON /){ + # there are two cases for abandon + # [17/Sep/2013:01:52:11 -0400] conn=1482074 op=4 ABANDON targetop=3 msgid=4 + # [17/Sep/2013:01:52:11 -0400] conn=1482074 op=4 ABANDON targetop=NOTFOUND msgid=4 + # if the op to be abandoned completes before the abandon request can be processed + # the server will use NOTFOUND as the op number $abandonCount++; if($reportStats){ inc_stats('abandon',$s_stats,$m_stats); } $allResults++; - if ($_ =~ /targetop= *([0-9a-zA-Z]+)/i ){ - push @{$arrays->{targetop}}, $1; - if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{targetconn}}, $1; } - if ($_ =~ /msgid= *([0-9\-]+)/i){ push @{$arrays->{msgid}}, $1; } + if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+) +ABANDON +targetop= *([0-9a-zA-Z]+) +msgid= *([0-9\-]+)/i ){ + # abandon_conn_op - key is the conn,op of the ABANDON request + # the value is the targetop,msgid of the ABANDON request + $hashes->{abandon_conn_op}->{"$1,$2"} = "$3,$4"; # targetop,msgid } } if (m/ VLV /){ @@ -1738,11 +1683,11 @@ sub parseLineNormal $autobindCount++; $bindCount++; if($reportStats){ inc_stats('bind',$s_stats,$m_stats); } - if ($1 ne ""){ + if ($1 ne ""){ $tmpp = $1; $tmpp =~ tr/A-Z/a-z/; $hashes->{bindlist}->{$tmpp}++; - if($1 eq $rootDN){ + if($1 eq $rootDN){ $rootDNBindCount++; } } else { @@ -1752,7 +1697,7 @@ sub parseLineNormal } } if (m/ connection from/){ - if ($_ =~ /connection from *([0-9A-Fa-f\.\:]+)/i ){ + if ($_ =~ /connection from *([0-9A-Fa-f\.\:]+)/i ){ for (my $xxx =0; $xxx < $#excludeIP; $xxx++){ if ($excludeIP[$xxx] eq $1){$exc = "yes";} } @@ -1763,7 +1708,7 @@ sub parseLineNormal } $simConnection++; if ($simConnection > $maxsimConnection) { - $maxsimConnection = $simConnection; + $maxsimConnection = $simConnection; } ($connID) = $_ =~ /conn=(\d*)\s/; $openConnection[$connID]++; @@ -1772,7 +1717,7 @@ sub parseLineNormal my ($date, $hr, $min, $sec) = split (':', $time); my ($day, $mon, $yr) = split ('/', $date); $day =~ s/\[//; - $start_time_of_connection[$connID] = timegm($sec, $min, $hr, $day, $monthname{$mon}, $yr); + $hashes->{start_time_of_connection}->{$connID} = timegm($sec, $min, $hr, $day, $monthname{$mon}, $yr); } } if (m/ SSL client bound as /){$sslClientBindCount++;} @@ -1788,10 +1733,10 @@ sub parseLineNormal # if we didn't see the start time of this connection # i.e. due to truncation or log rotation # then just set to 0 - my $stoc = $start_time_of_connection[$connID] || 0; - $end_time_of_connection[$connID] = $gmtime || 0; - my $diff = $end_time_of_connection[$connID] - $stoc; - $start_time_of_connection[$connID] = $end_time_of_connection[$connID] = 0; + my $stoc = $hashes->{start_time_of_connection}->{$connID} || 0; + $hashes->{end_time_of_connection}->{$connID} = $gmtime || 0; + my $diff = $hashes->{end_time_of_connection}->{$connID} - $stoc; + $hashes->{start_time_of_connection}->{$connID} = $hashes->{end_time_of_connection}->{$connID} = 0; if ($diff <= 1) { $latency[0]++;} if ($diff == 2) { $latency[1]++;} if ($diff == 3) { $latency[2]++;} @@ -1802,85 +1747,82 @@ sub parseLineNormal } } if (m/ BIND/ && $_ =~ /dn=\"(.*)\" method/i ){ + my $binddn = $1; if($reportStats){ inc_stats('bind',$s_stats,$m_stats); } $bindCount++; - if ($1 ne ""){ - if($1 eq $rootDN){$rootDNBindCount++;} - $tmpp = $1; + my ($conn, $op); + if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ + $conn = $1; + $op = $2; + } + if ($binddn ne ""){ + if($binddn eq $rootDN){$rootDNBindCount++;} + $tmpp = $binddn; $tmpp =~ tr/A-Z/a-z/; $hashes->{bindlist}->{$tmpp}++; - if ($_ =~ /conn= *([0-9A-Z]+)/i) { push @{$arrays->{bindconn}}, $1;} - if ($_ =~ /op= *([0-9\-]+)/i) { push @{$arrays->{bindop}}, $1;} if($usage =~ /f/ || $verb eq "yes"){ - push @{$arrays->{binddn}}, $tmpp; + $hashes->{bind_conn_op}->{"$conn,$op"} = $tmpp; } } else { $anonymousBindCount++; $hashes->{bindlist}->{"Anonymous Binds"}++; - if ($_ =~ /conn= *([0-9A-Z]+)/i) { push @{$arrays->{bindconn}}, $1;} - if ($_ =~ /op= *([0-9\-]+)/i) { push @{$arrays->{bindop}}, $1;} - push @{$arrays->{binddn}}, ""; + if($usage =~ /f/ || $verb eq "yes"){ + $hashes->{bind_conn_op}->{"$conn,$op"} = ""; + } inc_stats('anonbind',$s_stats,$m_stats); } } if (m/ UNBIND/){ $unbindCount++; if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{unbindconn}}, $1; } - if ($_ =~ /op= *([0-9\-]+)/i){ push @{$arrays->{unbindop}}, $1; } + if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{unbind_conn_op}->{"$1,$2"}++;} } } - if (m/ RESULT err=/ && m/ notes=P/){ + if (m/ RESULT err=/ && m/ notes=[A-Z,]*P/){ $pagedSearchCount++; } - if (m/ notes=A/){ + if (m/ notes=[A-Z,]*A/){ $con = ""; if ($_ =~ /conn= *([0-9A-Z]+)/i){ - $con = $1; - if ($_ =~ /op= *([0-9\-]+)/i){ $op = $1;} + $con = $1; + if ($_ =~ /op= *([0-9\-]+)/i){ $op = $1;} } for (my $i=0; $i < $vlvCount;$i++){ - if ($vlvconn[$i] eq $con && $vlvop[$i] eq $op){ $vlvNotesACount++; $isVlvNotes="1";} + if ($vlvconn[$i] eq $con && $vlvop[$i] eq $op){ $vlvNotesACount++; $isVlvNotes="1";} } if($isVlvNotes == 0){ # We don't want to record vlv unindexed searches for our regular "bad" # unindexed search stat, as VLV unindexed searches aren't that bad $unindexedSrchCountNotesA++; if($reportStats){ inc_stats('notesA',$s_stats,$m_stats); } - } - if ($usage =~ /u/ || $verb eq "yes"){ - if ($isVlvNotes == 0 ){ - if ($_ =~ /etime= *([0-9.]+)/i ){ push @{$arrays->{notesAetime}}, $1; } - if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{notesAconn}}, $1; } - if ($_ =~ /op= *([0-9\-]+)/i){ push @{$arrays->{notesAop}}, $1; } - if ($_ =~ / *([0-9a-z:\/]+)/i){ push @{$arrays->{notesAtime}}, $1; } - if ($_ =~ /nentries= *([0-9]+)/i ){ push @{$arrays->{notesAnentries}}, $1; } + if ($usage =~ /u/ || $verb eq "yes"){ + $hashes->{notesa_conn_op}->{"$con,$op"}++; + if ($_ =~ /etime= *([0-9.]+)/i ){ $hashes->{etime_conn_op}->{"$con,$op"} = $1; } + if ($_ =~ / *([0-9a-z:\/]+)/i){ $hashes->{time_conn_op}->{"$con,$op"} = $1; } + if ($_ =~ /nentries= *([0-9]+)/i ){ $hashes->{nentries_conn_op}->{"$con,$op"} = $1; } } } $isVlvNotes = 0; } - if (m/ notes=U/){ + if (m/ notes=[A-Z,]*U/){ $con = ""; if ($_ =~ /conn= *([0-9A-Z]+)/i){ - $con = $1; - if ($_ =~ /op= *([0-9\-]+)/i){ $op = $1;} + $con = $1; + if ($_ =~ /op= *([0-9\-]+)/i){ $op = $1;} } for (my $i=0; $i < $vlvCount;$i++){ - if ($vlvconn[$i] eq $con && $vlvop[$i] eq $op){ $vlvNotesUCount++; $isVlvNotes="1";} + if ($vlvconn[$i] eq $con && $vlvop[$i] eq $op){ $vlvNotesUCount++; $isVlvNotes="1";} } if($isVlvNotes == 0){ # We don't want to record vlv unindexed searches for our regular "bad" # unindexed search stat, as VLV unindexed searches aren't that bad $unindexedSrchCountNotesU++; if($reportStats){ inc_stats('notesU',$s_stats,$m_stats); } - } - if ($usage =~ /u/ || $verb eq "yes"){ - if ($isVlvNotes == 0 ){ - if ($_ =~ /etime= *([0-9.]+)/i ){ push @{$arrays->{notesUetime}}, $1; } - if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{notesUconn}}, $1; } - if ($_ =~ /op= *([0-9\-]+)/i){ push @{$arrays->{notesUop}}, $1; } - if ($_ =~ / *([0-9a-z:\/]+)/i){ push @{$arrays->{notesUtime}}, $1; } - if ($_ =~ /nentries= *([0-9]+)/i ){ push @{$arrays->{notesUnentries}}, $1; } + if ($usage =~ /u/ || $verb eq "yes"){ + $hashes->{notesu_conn_op}->{"$con,$op"}++; + if ($_ =~ /etime= *([0-9.]+)/i ){ $hashes->{etime_conn_op}->{"$con,$op"} = $1; } + if ($_ =~ / *([0-9a-z:\/]+)/i){ $hashes->{time_conn_op}->{"$con,$op"} = $1; } + if ($_ =~ /nentries= *([0-9]+)/i ){ $hashes->{nentries_conn_op}->{"$con,$op"} = $1; } } } $isVlvNotes = 0; @@ -1941,8 +1883,8 @@ sub parseLineNormal } $ip = $1; $hashes->{ip_hash}->{$ip}++; - if ($_ =~ /conn= *([0-9A-Z]+)/i ){ - if ($exc ne "yes"){ + if ($_ =~ /conn= *([0-9A-Z]+)/i ){ + if ($exc ne "yes"){ $hashes->{conn_hash}->{$1} = $ip; } } @@ -1951,7 +1893,6 @@ sub parseLineNormal if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; $ip = getIPfromConn($1); - if ($ip eq ""){$ip = "Unknown_Host";} for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } @@ -1966,14 +1907,13 @@ sub parseLineNormal if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; $ip = getIPfromConn($1); - if ($ip eq ""){$ip = "Unknown_Host";} for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ - if ($ip eq $excludeIP[$xxx]){$exc = "yes";} + if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } if ($exc ne "yes"){ $hashes->{B1}->{$ip}++; $hashes->{conncount}->{"B1"}++; - $connCodeCount++; + $connCodeCount++; } } } @@ -1981,44 +1921,41 @@ sub parseLineNormal if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; $ip = getIPfromConn($1); - if ($ip eq ""){$ip = "Unknown_Host";} for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ - if ($ip eq $excludeIP[$xxx]){$exc = "yes";} + if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } if ($exc ne "yes"){ $hashes->{B4}->{$ip}++; $hashes->{conncount}->{"B4"}++; $connCodeCount++; } - } + } } if (m/- T1/){ if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; - $ip = getIPfromConn($1); - if ($ip eq ""){$ip = "Unknown_Host";} + $ip = getIPfromConn($1); for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ - if ($ip eq $excludeIP[$xxx]){$exc = "yes";} + if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } if ($exc ne "yes"){ $hashes->{T1}->{$ip}++; $hashes->{conncount}->{"T1"}++; - $connCodeCount++; + $connCodeCount++; } } } if (m/- T2/){ if ($_ =~ /conn= *([0-9A-Z]+)/i) { - $exc = "no"; + $exc = "no"; $ip = getIPfromConn($1); - if ($ip eq ""){$ip = "Unknown_Host";} for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ - if ($ip eq $excludeIP[$xxx]){$exc = "yes";} + if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } if ($exc ne "yes"){ $hashes->{T2}->{$ip}++; $hashes->{conncount}->{"T2"}++; - $connCodeCount++; + $connCodeCount++; } } } @@ -2027,14 +1964,13 @@ sub parseLineNormal $exc = "no"; $ip = getIPfromConn($1); $maxBerSizeCount++; - if ($ip eq ""){$ip = "Unknown_Host";} for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ - if ($ip eq $excludeIP[$xxx]){$exc = "yes";} + if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } if ($exc ne "yes"){ $hashes->{B2}->{$ip}++; $hashes->{conncount}->{"B2"}++; - $connCodeCount++; + $connCodeCount++; } } } @@ -2042,14 +1978,13 @@ sub parseLineNormal if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; $ip = getIPfromConn($1); - if ($ip eq ""){$ip = "Unknown_Host";} for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ - if ($ip eq $excludeIP[$xxx]){$exc = "yes";} + if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } if ($exc ne "yes"){ $hashes->{B3}->{$ip}++; $hashes->{conncount}->{"B3"}++; - $connCodeCount++; + $connCodeCount++; } } } @@ -2057,9 +1992,8 @@ sub parseLineNormal if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; $ip = getIPfromConn($1); - if ($ip eq ""){$ip = "Unknown_Host";} for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ - if ($ip eq $excludeIP[$xxx]){$exc = "yes";} + if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } if ($exc ne "yes"){ $hashes->{R1}->{$ip}++; @@ -2072,14 +2006,13 @@ sub parseLineNormal if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; $ip = getIPfromConn($1); - if ($ip eq ""){$ip = "Unknown_Host";} for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ - if ($ip eq $excludeIP[$xxx]){$exc = "yes";} + if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } if ($exc ne "yes"){ $hashes->{P1}->{$ip}++; $hashes->{conncount}->{"P1"}++; - $connCodeCount++; + $connCodeCount++; } } } @@ -2087,9 +2020,8 @@ sub parseLineNormal if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; $ip = getIPfromConn($1); - if ($ip eq ""){$ip = "Unknown_Host";} for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ - if ($ip eq $excludeIP[$xxx]){$exc = "yes";} + if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } if ($exc ne "yes"){ $hashes->{P2}->{$ip}++; @@ -2102,9 +2034,8 @@ sub parseLineNormal if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; $ip = getIPfromConn($1); - if ($ip eq ""){$ip = "Unknown_Host";} for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ - if ($ip eq $excludeIP[$xxx]){$exc = "yes";} + if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } if ($exc ne "yes"){ $hashes->{U1}->{$ip}++; @@ -2131,8 +2062,7 @@ sub parseLineNormal if ($_ =~ /oid=\" *([0-9\.]+)/i ){ $hashes->{oid}->{$1}++; } if ($1 && $1 eq $startTLSoid){$startTLSCount++;} if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{extconn}}, $1; } - if ($_ =~ /op= *([0-9\-]+)/i){ push @{$arrays->{extop}}, $1; } + if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{ext_conn_op}->{"$1,$2"}++;} } } if (($usage =~ /l/ || $verb eq "yes") and / SRCH /){ @@ -2155,14 +2085,12 @@ sub parseLineNormal $filterCount++; if($usage =~ /u/ || $verb eq "yes"){ # we only need this for the unindexed search report - push @{$arrays->{filterval}}, $tmpp; - push @{$arrays->{filterconn}}, $filterConn; - push @{$arrays->{filterop}}, $filterOp; + $hashes->{filter_conn_op}->{"$filterConn,$filterOp"} = $tmpp; } } if ($usage =~ /a/ || $verb eq "yes"){ if (/ SRCH / && $_ =~ /base=\"(.*)\" scope/i ){ - my ($baseConn, $baseOp, $scopeVal, $scopeConn, $scopeOp); + my ($conn, $op, $scopeVal); if ($1 eq ""){ $tmpp = "Root DSE"; } else { @@ -2173,25 +2101,19 @@ sub parseLineNormal # # grab the search bases & scope for potential unindexed searches # - if ($_ =~ /scope= *([0-9]+)/i) { - $scopeVal = $1; + if ($_ =~ /scope= *([0-9]+)/i) { + $scopeVal = $1; } - if ($_ =~ /conn= *([0-9A-Z]+)/i) { - $baseConn = $1; - $scopeConn = $1; + if ($_ =~ /conn= *([0-9A-Z]+)/i) { + $conn = $1; } - if ($_ =~ /op= *([0-9\-]+)/i) { - $baseOp = $1; - $scopeOp = $1; + if ($_ =~ /op= *([0-9\-]+)/i) { + $op = $1; } if($usage =~ /u/ || $verb eq "yes"){ # we only need this for the unindexed search report - push @{$arrays->{baseval}}, $tmpp; - push @{$arrays->{baseconn}}, $baseConn; - push @{$arrays->{baseop}}, $baseOp; - push @{$arrays->{scopeval}}, $scopeTxt[$scopeVal]; - push @{$arrays->{scopeconn}}, $scopeConn; - push @{$arrays->{scopeop}}, $scopeOp; + $hashes->{base_conn_op}->{"$conn,$op"} = $tmpp; + $hashes->{scope_conn_op}->{"$conn,$op"} = $scopeTxt[$scopeVal]; } $baseCount++; $scopeCount++; @@ -2210,15 +2132,8 @@ sub parseLineNormal $ds6x = "true"; $badPwdCount++; } elsif (/ err=49 tag=/ ){ - if ($_ =~ /conn= *([0-9A-Z]+)/i ){ - push @{$arrays->{badpwdconn}}, $1; - $ip = getIPfromConn($1); - $badPwdCount++; - } - if ($_ =~ /op= *([0-9\-]+)/i ){ - push @{$arrays->{badpwdop}}, $1; - } - push @{$arrays->{badpwdip}}, $ip; + $badPwdCount++; + if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{badpwd_conn_op}->{"$1,$2"}++;} } } if (/ BIND / && /method=sasl/i){ @@ -2259,29 +2174,17 @@ sub parseLineNormal sub reset_stats_block { - my $stats = shift; - - $stats->{'last'} = shift || 0; - $stats->{'last_str'} = shift || ''; - - $stats->{'results'}=0; - $stats->{'srch'}=0; - $stats->{'add'}=0; - $stats->{'mod'}=0; - $stats->{'modrdn'}=0; - $stats->{'moddn'}=0; - $stats->{'cmp'}=0; - $stats->{'del'}=0; - $stats->{'abandon'}=0; - $stats->{'conns'}=0; - $stats->{'sslconns'}=0; - $stats->{'bind'}=0; - $stats->{'anonbind'}=0; - $stats->{'unbind'}=0; - $stats->{'notesA'}=0; - $stats->{'notesU'}=0; - $stats->{'etime'}=0; - return; + my $stats = shift; + + $stats->{'last'} = shift || 0; + $stats->{'last_str'} = shift || ''; + + for my $sn (@statnames) { + next if ($sn eq 'last' or $sn eq 'last_str'); + $stats->{$sn}=0; + } + + return; } sub @@ -2292,6 +2195,7 @@ new_stats_block 'active' => 0, }; if ($name){ + $stats = openHashFiles($dataLocation, @statnames); $stats->{'filename'} = $name; $stats->{'fh'} = new IO::File; $stats->{'active'} = open($stats->{'fh'},">$name"); @@ -2308,32 +2212,32 @@ print_stats_block if ($stats->{'active'}){ if ($stats->{'last'}){ $stats->{'fh'}->print( - join(',', - $stats->{'last_str'}, - $stats->{'last'}, - $stats->{'results'}, - $stats->{'srch'}, - $stats->{'add'}, - $stats->{'mod'}, - $stats->{'modrdn'}, - $stats->{'moddn'}, - $stats->{'cmp'}, - $stats->{'del'}, - $stats->{'abandon'}, - $stats->{'conns'}, - $stats->{'sslconns'}, - $stats->{'bind'}, - $stats->{'anonbind'}, - $stats->{'unbind'}, - $stats->{'notesA'}, - $stats->{'notesU'}, - $stats->{'etime'}), - "\n" ); + join(',', + $stats->{'last_str'}, + $stats->{'last'}, + $stats->{'results'}, + $stats->{'srch'}, + $stats->{'add'}, + $stats->{'mod'}, + $stats->{'modrdn'}, + $stats->{'moddn'}, + $stats->{'cmp'}, + $stats->{'del'}, + $stats->{'abandon'}, + $stats->{'conns'}, + $stats->{'sslconns'}, + $stats->{'bind'}, + $stats->{'anonbind'}, + $stats->{'unbind'}, + $stats->{'notesA'}, + $stats->{'notesU'}, + $stats->{'etime'}), + "\n" ); } else { $stats->{'fh'}->print( - "Time,time_t,Results,Search,Add,Mod,Modrdn,Moddn,Compare,Delete,Abandon,". - "Connections,SSL Conns,Bind,Anon Bind,Unbind,Unindexed search,Unindexed component,ElapsedTime\n" - ); + "Time,time_t,Results,Search,Add,Mod,Modrdn,Moddn,Compare,Delete,Abandon,". + "Connections,SSL Conns,Bind,Anon Bind,Unbind,Unindexed search,Unindexed component,ElapsedTime\n" + ); } } } @@ -2346,7 +2250,7 @@ inc_stats my $n = shift; foreach(@_){ $_->{$n}++ - if exists $_->{$n}; + if exists $_->{$n}; } return; } @@ -2360,7 +2264,7 @@ inc_stats_val my $val = shift; foreach(@_){ $_->{$n} += $val - if exists $_->{$n}; + if exists $_->{$n}; } return; } @@ -2383,7 +2287,7 @@ displayBindReport &printClients($bindReport{$bindDN}{"conn"}); print("\n Operations Performed:\n\n"); &printOpStats($bindDN); - print("\n"); + print("\n"); } print "Done.\n"; exit (0); @@ -2391,13 +2295,13 @@ displayBindReport sub printClients -{ +{ my @bindConns = &cleanConns(split(' ', $_[0])); my $IPcount = "1"; foreach my $ip ( keys %connList ){ # Loop over all the IP addresses foreach my $bc (@bindConns){ # Loop over each bind conn number and compare it - if($connList{$ip} =~ / $bc /){ + if($connList{$ip} =~ / $bc /){ print(" [$IPcount] $ip\n"); $IPcount++; last; @@ -2417,7 +2321,7 @@ cleanConns if($dirtyConns[$i] ne ""){ $retConns[$c++] = $dirtyConns[$i]; } - } + } return @retConns; } @@ -2443,7 +2347,7 @@ printOpStats ####################### # # # Hash File Functions # -# # +# # ####################### sub @@ -2471,35 +2375,17 @@ openHashFiles } sub -openArrayFiles -{ - my $dir = shift; - my %arrays = (); - for my $an (@_) { - my @ary = (); # using my in inner loop will create brand new array every time through for tie - my $fn = "$dir/$an.logconv.db"; - push @removefiles, $fn; - tie @ary, "DB_File", $fn, O_CREAT|O_RDWR, 0600, $DB_RECNO or do { openFailed($!, $fn) }; - $arrays{$an} = \@ary; - } - return \%arrays; -} - -sub removeDataFiles { - if (!$needCleanup) { return ; } + if (!$needCleanup) { return ; } for my $h (keys %{$hashes}) { untie %{$hashes->{$h}}; } - for my $a (keys %{$arrays}) { - untie @{$arrays->{$a}}; - } for my $file (@removefiles) { unlink $file; } - $needCleanup = 0; + $needCleanup = 0; } END { print "Cleaning up temp files . . .\n"; removeDataFiles(); print "Done\n"; } @@ -2508,7 +2394,12 @@ sub getIPfromConn { my $connid = shift; - return $hashes->{conn_hash}->{$connid}; + if (exists($hashes->{conn_hash}->{$connid}) && + defined($hashes->{conn_hash}->{$connid})) { + return $hashes->{conn_hash}->{$connid}; + } + + return "Unknown_Host"; } ####################################### commit de5a6277590692fb2610c23f88e5546363764683 Author: Mark Reynolds <mreynolds(a)redhat.com&gt; Date: Mon Sep 23 11:32:16 2013 -0400 Ticket 47520 - Fix various issues with logconv.pl Bug Description: When testing an access log that contains "internal" connections/ops, many warning messages are created. Other warnings are generated because the DB_File arrays need to have thier elements checked before comparing to other strings. Fix Description: Improve value checking. Add support for connection numbers that are words (Internal), and negative values for op ids(-1). https://fedorahosted.org/389/ticket/47520 Reviewed by: richm(Thanks!) (cherry picked from commit 8b153e999ac6dc4869498a78da103a07db49d28e) (cherry picked from commit 2c5d52b7a4aef8317cc9e34a9b1153c74620010d) diff --git a/ldap/admin/src/logconv.pl b/ldap/admin/src/logconv.pl index ec0fdf5..3a6fa01 100755 --- a/ldap/admin/src/logconv.pl +++ b/ldap/admin/src/logconv.pl @@ -102,6 +102,7 @@ my $fdds = 0; my $reportBinds = "no"; my $rootDN = ""; my $needCleanup = 0; +my @scopeTxt = ("0 (base)", "1 (one)", "2 (subtree)"); GetOptions( 'd|rootDN=s' => \$rootDN, @@ -653,6 +654,7 @@ print "Entire Search Base Queries: $objectclassTopCount\n"; print "Paged Searches: $pagedSearchCount\n"; print "Unindexed Searches: $unindexedSrchCountNotesA\n"; print "Unindexed Components: $unindexedSrchCountNotesU\n"; + if ($verb eq "yes" || $usage =~ /u/){ if ($unindexedSrchCountNotesA > 0){ my $conn_hash = $hashes->{conn_hash}; @@ -670,11 +672,12 @@ if ($verb eq "yes" || $usage =~ /u/){ my $filter_val = $arrays->{filterval}; my $filter_conn = $arrays->{filterconn}; my $filter_op = $arrays->{filterop}; - my $notesCount = "1"; my $unindexedIp; + for (my $n = 0; $n < scalar(@{$notesEtime}); $n++){ - if($conn_hash->{$notesConn->[$n]} eq ""){ + if(!$notesConn->[$n] || $notesConn->[$n] eq "" || + !$conn_hash->{$notesConn->[$n]} || $conn_hash->{$notesConn->[$n]} eq ""){ $unindexedIp = "?"; } else { $unindexedIp = $conn_hash->{$notesConn->[$n]}; @@ -688,18 +691,21 @@ if ($verb eq "yes" || $usage =~ /u/){ print " - IP Address: $unindexedIp\n"; for (my $nnn = 0; $nnn < $baseCount; $nnn++){ + if(!$base_conn->[$nnn] || !$base_op->[$nnn]){ next; } if ($notesConn->[$n] eq $base_conn->[$nnn] && $notesOp->[$n] eq $base_op->[$nnn]){ print " - Search Base: $base_val->[$nnn]\n"; last; } } for (my $nnn = 0; $nnn < $scopeCount; $nnn++){ + if(!$scope_conn->[$nnn] || !$scope_op->[$nnn]){ next; } if ($notesConn->[$n] eq $scope_conn->[$nnn] && $notesOp->[$n] eq $scope_op->[$nnn]){ print " - Search Scope: $scope_val->[$nnn]\n"; last; } } for (my $nnn = 0; $nnn < $filterCount; $nnn++){ + if(!$filter_conn->[$nnn] || !$filter_op->[$nnn]){ next; } if ($notesConn->[$n] eq $filter_conn->[$nnn] && $notesOp->[$n] eq $filter_op->[$nnn]){ print " - Search Filter: $filter_val->[$nnn]\n"; last; @@ -727,7 +733,8 @@ if ($verb eq "yes" || $usage =~ /u/){ my $notesCount = "1"; my $unindexedIp; for (my $n = 0; $n < scalar(@{$notesEtime}); $n++){ - if($conn_hash->{$notesConn->[$n]} eq ""){ + if(!$notesConn->[$n] || $notesConn->[$n] eq "" || + !$conn_hash->{$notesConn->[$n]} || $conn_hash->{$notesConn->[$n]} eq ""){ $unindexedIp = "?"; } else { $unindexedIp = $conn_hash->{$notesConn->[$n]}; @@ -741,18 +748,21 @@ if ($verb eq "yes" || $usage =~ /u/){ print " - IP Address: $unindexedIp\n"; for (my $nnn = 0; $nnn < $baseCount; $nnn++){ + if(!$base_conn->[$nnn] || !$base_op->[$nnn]){ next; } if ($notesConn->[$n] eq $base_conn->[$nnn] && $notesOp->[$n] eq $base_op->[$nnn]){ print " - Search Base: $base_val->[$nnn]\n"; last; } } for (my $nnn = 0; $nnn < $scopeCount; $nnn++){ + if(!$scope_conn->[$nnn] || !$scope_op->[$nnn]){ next; } if ($notesConn->[$n] eq $scope_conn->[$nnn] && $notesOp->[$n] eq $scope_op->[$nnn]){ print " - Search Scope: $scope_val->[$nnn]\n"; last; } } for (my $nnn = 0; $nnn < $filterCount; $nnn++){ + if(!$filter_conn->[$nnn] || !$filter_op->[$nnn]){ next; } if ($notesConn->[$n] eq $filter_conn->[$nnn] && $notesOp->[$n] eq $filter_op->[$nnn]){ print " - Search Filter: $filter_val->[$nnn]\n"; last; @@ -772,13 +782,12 @@ if ($brokenPipeCount > 0){ my @etext; foreach my $key (sort { $rc->{$b} <=> $rc->{$a} } keys %{$rc}) { if ($rc->{$key} > 0){ - if ($conn{$key} eq ""){$conn{$key} = "**Unknown**";} - push @etext, sprintf " - %-4s (%2s) %-40s\n",$rc->{$key},$conn{$key},$connmsg{$key -}; - } - } - print @etext; - print "\n"; + if ($conn{$key} eq ""){$conn{$key} = "**Unknown**";} + push @etext, sprintf " - %-4s (%2s) %-40s\n",$rc->{$key},$conn{$key},$connmsg{$key}; + } + } + print @etext; + print "\n"; } print "Connections Reset By Peer: $connResetByPeerCount\n"; @@ -787,12 +796,11 @@ if ($connResetByPeerCount > 0){ my @retext; foreach my $key (sort { $src->{$b} <=> $src->{$a} } keys %{$src}) { if ($src->{$key} > 0){ - if ($conn{$key} eq ""){$conn{$key} = "**Unknown**";} - push @retext, sprintf " - %-4s (%2s) %-40s\n",$src->{$key},$conn{$key},$connmsg{$key -}; - } - } - print @retext; + if ($conn{$key} eq ""){$conn{$key} = "**Unknown**";} + push @retext, sprintf " - %-4s (%2s) %-40s\n",$src->{$key},$conn{$key},$connmsg{$key}; + } + } + print @retext; print "\n"; } @@ -802,9 +810,9 @@ if ($resourceUnavailCount > 0){ my @rtext; foreach my $key (sort { $rsrc->{$b} <=> $rsrc->{$a} } keys %{$rsrc}) { if ($rsrc->{$key} > 0){ - if ($conn{$key} eq ""){$conn{$key} = "**Resource Issue**";} - push @rtext, sprintf " - %-4s (%2s) %-40s\n",$rsrc->{$key},$conn{$key},$connmsg{$key}; - } + if ($conn{$key} eq ""){$conn{$key} = "**Resource Issue**";} + push @rtext, sprintf " - %-4s (%2s) %-40s\n",$rsrc->{$key},$conn{$key},$connmsg{$key}; + } } print @rtext; } @@ -1244,48 +1252,57 @@ if ($usage =~ /g/i || $verb eq "yes"){ my $op = $arrays->{targetop}->[$g]; my $msgid = $arrays->{msgid}->[$g]; for (my $sc = 0; $sc < $srchCount; $sc++){ - if ($arrays->{srchconn}->[$sc] eq $conn && $arrays->{srchop}->[$sc] eq $op ){ + if (($arrays->{srchconn}->[$sc] && $arrays->{srchop}->[$sc]) && + ($arrays->{srchconn}->[$sc] eq $conn && $arrays->{srchop}->[$sc] eq $op )){ print " - SRCH conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; } } for (my $dc = 0; $dc < $delCount; $dc++){ - if ($arrays->{delconn}->[$dc] eq $conn && $arrays->{delop}->[$dc] eq $op){ - print " - DEL conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; + if (($arrays->{delconn}->[$dc] && $arrays->{delop}->[$dc]) && + ($arrays->{delconn}->[$dc] eq $conn && $arrays->{delop}->[$dc] eq $op)){ + print " - DEL conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; } } for (my $adc = 0; $adc < $addCount; $adc++){ - if ($arrays->{addconn}->[$adc] eq $conn && $arrays->{addop}->[$adc] eq $op){ - print " - ADD conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; + if (($arrays->{addconn}->[$adc] && $arrays->{addop}->[$adc]) && + ($arrays->{addconn}->[$adc] eq $conn && $arrays->{addop}->[$adc] eq $op)){ + print " - ADD conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; } } for (my $mc = 0; $mc < $modCount; $mc++){ - if ($arrays->{modconn}->[$mc] eq $conn && $arrays->{modop}->[$mc] eq $op){ - print " - MOD conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; + if (($arrays->{modconn}->[$mc] && $arrays->{modop}->[$mc]) && + ($arrays->{modconn}->[$mc] eq $conn && $arrays->{modop}->[$mc] eq $op)){ + print " - MOD conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; } } for (my $cc = 0; $cc < $cmpCount; $cc++){ - if ($arrays->{cmpconn}->[$cc] eq $conn && $arrays->{cmpop}->[$cc] eq $op){ - print " - CMP conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; + if (($arrays->{cmpconn}->[$cc] && $arrays->{cmpop}->[$cc]) && + ($arrays->{cmpconn}->[$cc] eq $conn && $arrays->{cmpop}->[$cc] eq $op)){ + print " - CMP conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; } } for (my $mdc = 0; $mdc < $modrdnCount; $mdc++){ - if ($arrays->{modrdnconn}->[$mdc] eq $conn && $arrays->{modrdnop}->[$mdc] eq $op){ + if (($arrays->{modrdnconn}->[$mdc] && $arrays->{modrdnop}->[$mdc]) && + ($arrays->{modrdnconn}->[$mdc] eq $conn && $arrays->{modrdnop}->[$mdc] eq $op)){ print " - MODRDN conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; } } for (my $bcb = 0; $bcb < $bindCount; $bcb++){ - if ($arrays->{bindconn}->[$bcb] eq $conn && $arrays->{bindop}->[$bcb] eq $op){ + if (($arrays->{bindconn}->[$bcb] && $arrays->{bindop}->[$bcb]) && + ($arrays->{bindconn}->[$bcb] eq $conn && $arrays->{bindop}->[$bcb] eq $op)){ print " - BIND conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; } } for (my $ubc = 0; $ubc < $unbindCount; $ubc++){ - if ($arrays->{unbindconn}->[$ubc] eq $conn && $arrays->{unbindop}->[$ubc] eq $op){ + if (($arrays->{unbindconn}->[$ubc] && $arrays->{unbindop}->[$ubc]) && + ($arrays->{unbindconn}->[$ubc] eq $conn && $arrays->{unbindop}->[$ubc] eq $op)){ print " - UNBIND conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; } } for (my $ec = 0; $ec < $extopCount; $ec++){ - if ($arrays->{extconn}->[$ec] eq $conn && $arrays->{extop}->[$ec] eq $op){ - print " - EXT conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; + if (($arrays->{extconn}->[$ec] && $arrays->{extop}->[$ec]) && + ($arrays->{extconn}->[$ec] eq $conn && $arrays->{extop}->[$ec] eq $op)){ + print " - EXT conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; } } } @@ -1485,7 +1502,7 @@ parseLineBind { return ; } $ip = $1; - if ($_ =~ /conn= *([0-9]+)/i ){ + if ($_ =~ /conn= *([0-9A-Z]+)/i ){ $connList{$ip} = $connList{$ip} . " $1 "; } return; @@ -1516,7 +1533,7 @@ parseLineBind { $bindReport{$dn}{"modrdn"} = 0; $bindReport{$dn}{"failedBind"} = 0; } - if ($_ =~ /conn= *([0-9]+)/i) { + if ($_ =~ /conn= *([0-9A-Z]+)/i) { $bindReport{$dn}{"conn"} = $bindReport{$dn}{"conn"} . " $1 "; } return; @@ -1547,7 +1564,7 @@ processOpForBindReport my $op = shift; my $data = shift; - if ($data =~ /conn= *([0-9]+)/i) { + if ($data =~ /conn= *([0-9A-Z]+)/i) { foreach my $dn (keys %bindReport){ if ($bindReport{$dn}{"conn"} =~ / $1 /){ $bindReport{$dn}{$op}++; @@ -1649,48 +1666,48 @@ sub parseLineNormal $anyAttrs++; } if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{srchconn}}, $1;} - if ($_ =~ /op= *([0-9]+)/i){ push @{$arrays->{srchop}}, $1;} + if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{srchconn}}, $1;} + if ($_ =~ /op= *([0-9\-]+)/i){ push @{$arrays->{srchop}}, $1;} } } if (m/ DEL/){ $delCount++; if($reportStats){ inc_stats('del',$s_stats,$m_stats); } if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{delconn}}, $1;} - if ($_ =~ /op= *([0-9]+)/i){ push @{$arrays->{delop}}, $1;} + if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{delconn}}, $1;} + if ($_ =~ /op= *([0-9\-]+)/i){ push @{$arrays->{delop}}, $1;} } } if (m/ MOD dn=/){ $modCount++; if($reportStats){ inc_stats('mod',$s_stats,$m_stats); } if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{modconn}}, $1;} - if ($_ =~ /op= *([0-9]+)/i){ push @{$arrays->{modop}}, $1; } + if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{modconn}}, $1;} + if ($_ =~ /op= *([0-9\-]+)/i){ push @{$arrays->{modop}}, $1; } } } if (m/ ADD/){ $addCount++; if($reportStats){ inc_stats('add',$s_stats,$m_stats); } if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{addconn}}, $1; } - if ($_ =~ /op= *([0-9]+)/i){ push @{$arrays->{addop}}, $1; } + if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{addconn}}, $1; } + if ($_ =~ /op= *([0-9\-]+)/i){ push @{$arrays->{addop}}, $1; } } } if (m/ MODRDN/){ $modrdnCount++; if($reportStats){ inc_stats('modrdn',$s_stats,$m_stats); } if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{modrdnconn}}, $1; } - if ($_ =~ /op= *([0-9]+)/i){ push @{$arrays->{modrdnop}}, $1; } + if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{modrdnconn}}, $1; } + if ($_ =~ /op= *([0-9\-]+)/i){ push @{$arrays->{modrdnop}}, $1; } } } if (m/ CMP dn=/){ $cmpCount++; if($reportStats){ inc_stats('cmp',$s_stats,$m_stats); } if ($verb eq "yes" || $usage =~ /g/i){ - if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{cmpconn}}, $1;} - if ($_ =~ /op= *([0-9]+)/i){ push @{$arrays->{cmpop}}, $1;} + if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{cmpconn}}, $1;} + if ($_ =~ /op= *([0-9\-]+)/i){ push @{$arrays->{cmpop}}, $1;} } } if (m/ ABANDON /){ @@ -1699,13 +1716,13 @@ sub parseLineNormal $allResults++; if ($_ =~ /targetop= *([0-9a-zA-Z]+)/i ){ push @{$arrays->{targetop}}, $1; - if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{targetconn}}, $1; } - if ($_ =~ /msgid= *([0-9]+)/i){ push @{$arrays->{msgid}}, $1; } + if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{targetconn}}, $1; } + if ($_ =~ /msgid= *([0-9\-]+)/i){ push @{$arrays->{msgid}}, $1; } } } if (m/ VLV /){ - if ($_ =~ /conn= *([0-9]+)/i){ $vlvconn[$vlvCount] = $1;} - if ($_ =~ /op= *([0-9]+)/i){ $vlvop[$vlvCount] = $1;} + if ($_ =~ /conn= *([0-9A-Z]+)/i){ $vlvconn[$vlvCount] = $1;} + if ($_ =~ /op= *([0-9\-]+)/i){ $vlvop[$vlvCount] = $1;} $vlvCount++; } if (m/ authzid=/){ @@ -1792,16 +1809,16 @@ sub parseLineNormal $tmpp = $1; $tmpp =~ tr/A-Z/a-z/; $hashes->{bindlist}->{$tmpp}++; - if ($_ =~ /conn= *([0-9]+)/i) { push @{$arrays->{bindconn}}, $1;} - if ($_ =~ /op= *([0-9]+)/i) { push @{$arrays->{bindop}}, $1;} + if ($_ =~ /conn= *([0-9A-Z]+)/i) { push @{$arrays->{bindconn}}, $1;} + if ($_ =~ /op= *([0-9\-]+)/i) { push @{$arrays->{bindop}}, $1;} if($usage =~ /f/ || $verb eq "yes"){ push @{$arrays->{binddn}}, $tmpp; } } else { $anonymousBindCount++; $hashes->{bindlist}->{"Anonymous Binds"}++; - if ($_ =~ /conn= *([0-9]+)/i) { push @{$arrays->{bindconn}}, $1;} - if ($_ =~ /op= *([0-9]+)/i) { push @{$arrays->{bindop}}, $1;} + if ($_ =~ /conn= *([0-9A-Z]+)/i) { push @{$arrays->{bindconn}}, $1;} + if ($_ =~ /op= *([0-9\-]+)/i) { push @{$arrays->{bindop}}, $1;} push @{$arrays->{binddn}}, ""; inc_stats('anonbind',$s_stats,$m_stats); } @@ -1809,17 +1826,18 @@ sub parseLineNormal if (m/ UNBIND/){ $unbindCount++; if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{unbindconn}}, $1; } - if ($_ =~ /op= *([0-9]+)/i){ push @{$arrays->{unbindop}}, $1; } + if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{unbindconn}}, $1; } + if ($_ =~ /op= *([0-9\-]+)/i){ push @{$arrays->{unbindop}}, $1; } } } if (m/ RESULT err=/ && m/ notes=P/){ $pagedSearchCount++; } if (m/ notes=A/){ - if ($_ =~ /conn= *([0-9]+)/i){ + $con = ""; + if ($_ =~ /conn= *([0-9A-Z]+)/i){ $con = $1; - if ($_ =~ /op= *([0-9]+)/i){ $op = $1;} + if ($_ =~ /op= *([0-9\-]+)/i){ $op = $1;} } for (my $i=0; $i < $vlvCount;$i++){ if ($vlvconn[$i] eq $con && $vlvop[$i] eq $op){ $vlvNotesACount++; $isVlvNotes="1";} @@ -1833,8 +1851,8 @@ sub parseLineNormal if ($usage =~ /u/ || $verb eq "yes"){ if ($isVlvNotes == 0 ){ if ($_ =~ /etime= *([0-9.]+)/i ){ push @{$arrays->{notesAetime}}, $1; } - if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{notesAconn}}, $1; } - if ($_ =~ /op= *([0-9]+)/i){ push @{$arrays->{notesAop}}, $1; } + if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{notesAconn}}, $1; } + if ($_ =~ /op= *([0-9\-]+)/i){ push @{$arrays->{notesAop}}, $1; } if ($_ =~ / *([0-9a-z:\/]+)/i){ push @{$arrays->{notesAtime}}, $1; } if ($_ =~ /nentries= *([0-9]+)/i ){ push @{$arrays->{notesAnentries}}, $1; } } @@ -1842,9 +1860,10 @@ sub parseLineNormal $isVlvNotes = 0; } if (m/ notes=U/){ - if ($_ =~ /conn= *([0-9]+)/i){ + $con = ""; + if ($_ =~ /conn= *([0-9A-Z]+)/i){ $con = $1; - if ($_ =~ /op= *([0-9]+)/i){ $op = $1;} + if ($_ =~ /op= *([0-9\-]+)/i){ $op = $1;} } for (my $i=0; $i < $vlvCount;$i++){ if ($vlvconn[$i] eq $con && $vlvop[$i] eq $op){ $vlvNotesUCount++; $isVlvNotes="1";} @@ -1858,8 +1877,8 @@ sub parseLineNormal if ($usage =~ /u/ || $verb eq "yes"){ if ($isVlvNotes == 0 ){ if ($_ =~ /etime= *([0-9.]+)/i ){ push @{$arrays->{notesUetime}}, $1; } - if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{notesUconn}}, $1; } - if ($_ =~ /op= *([0-9]+)/i){ push @{$arrays->{notesUop}}, $1; } + if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{notesUconn}}, $1; } + if ($_ =~ /op= *([0-9\-]+)/i){ push @{$arrays->{notesUop}}, $1; } if ($_ =~ / *([0-9a-z:\/]+)/i){ push @{$arrays->{notesUtime}}, $1; } if ($_ =~ /nentries= *([0-9]+)/i ){ push @{$arrays->{notesUnentries}}, $1; } } @@ -1922,14 +1941,14 @@ sub parseLineNormal } $ip = $1; $hashes->{ip_hash}->{$ip}++; - if ($_ =~ /conn= *([0-9]+)/i ){ + if ($_ =~ /conn= *([0-9A-Z]+)/i ){ if ($exc ne "yes"){ $hashes->{conn_hash}->{$1} = $ip; } } } if (m/- A1/){ - if ($_ =~ /conn= *([0-9]+)/i) { + if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; $ip = getIPfromConn($1); if ($ip eq ""){$ip = "Unknown_Host";} @@ -1944,7 +1963,7 @@ sub parseLineNormal } } if (m/- B1/){ - if ($_ =~ /conn= *([0-9]+)/i) { + if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; $ip = getIPfromConn($1); if ($ip eq ""){$ip = "Unknown_Host";} @@ -1959,7 +1978,7 @@ sub parseLineNormal } } if (m/- B4/){ - if ($_ =~ /conn= *([0-9]+)/i) { + if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; $ip = getIPfromConn($1); if ($ip eq ""){$ip = "Unknown_Host";} @@ -1974,7 +1993,7 @@ sub parseLineNormal } } if (m/- T1/){ - if ($_ =~ /conn= *([0-9]+)/i) { + if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; $ip = getIPfromConn($1); if ($ip eq ""){$ip = "Unknown_Host";} @@ -1989,7 +2008,7 @@ sub parseLineNormal } } if (m/- T2/){ - if ($_ =~ /conn= *([0-9]+)/i) { + if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; $ip = getIPfromConn($1); if ($ip eq ""){$ip = "Unknown_Host";} @@ -2004,7 +2023,7 @@ sub parseLineNormal } } if (m/- B2/){ - if ($_ =~ /conn= *([0-9]+)/i) { + if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; $ip = getIPfromConn($1); $maxBerSizeCount++; @@ -2020,7 +2039,7 @@ sub parseLineNormal } } if (m/- B3/){ - if ($_ =~ /conn= *([0-9]+)/i) { + if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; $ip = getIPfromConn($1); if ($ip eq ""){$ip = "Unknown_Host";} @@ -2035,7 +2054,7 @@ sub parseLineNormal } } if (m/- R1/){ - if ($_ =~ /conn= *([0-9]+)/i) { + if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; $ip = getIPfromConn($1); if ($ip eq ""){$ip = "Unknown_Host";} @@ -2050,7 +2069,7 @@ sub parseLineNormal } } if (m/- P1/){ - if ($_ =~ /conn= *([0-9]+)/i) { + if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; $ip = getIPfromConn($1); if ($ip eq ""){$ip = "Unknown_Host";} @@ -2065,7 +2084,7 @@ sub parseLineNormal } } if (m/- P2/){ - if ($_ =~ /conn= *([0-9]+)/i) { + if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; $ip = getIPfromConn($1); if ($ip eq ""){$ip = "Unknown_Host";} @@ -2080,7 +2099,7 @@ sub parseLineNormal } } if (m/- U1/){ - if ($_ =~ /conn= *([0-9]+)/i) { + if ($_ =~ /conn= *([0-9A-Z]+)/i) { $exc = "no"; $ip = getIPfromConn($1); if ($ip eq ""){$ip = "Unknown_Host";} @@ -2112,8 +2131,8 @@ sub parseLineNormal if ($_ =~ /oid=\" *([0-9\.]+)/i ){ $hashes->{oid}->{$1}++; } if ($1 && $1 eq $startTLSoid){$startTLSCount++;} if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{extconn}}, $1; } - if ($_ =~ /op= *([0-9]+)/i){ push @{$arrays->{extop}}, $1; } + if ($_ =~ /conn= *([0-9A-Z]+)/i){ push @{$arrays->{extconn}}, $1; } + if ($_ =~ /op= *([0-9\-]+)/i){ push @{$arrays->{extop}}, $1; } } } if (($usage =~ /l/ || $verb eq "yes") and / SRCH /){ @@ -2123,15 +2142,15 @@ sub parseLineNormal $tmpp =~ tr/A-Z/a-z/; $tmpp =~ s/\\22/\"/g; $hashes->{filter}->{$tmpp}++; - if ($_ =~ /conn= *([0-9]+)/i) { $filterConn = $1; } - if ($_ =~ /op= *([0-9]+)/i) { $filterOp = $1; } + if ($_ =~ /conn= *([0-9A-Z]+)/i) { $filterConn = $1; } + if ($_ =~ /op= *([0-9\-]+)/i) { $filterOp = $1; } } elsif (/ SRCH / && $_ =~ /filter=\"(.*)\"/i){ $tmpp = $1; $tmpp =~ tr/A-Z/a-z/; $tmpp =~ s/\\22/\"/g; $hashes->{filter}->{$tmpp}++; - if ($_ =~ /conn= *([0-9]+)/i) { $filterConn = $1; } - if ($_ =~ /op= *([0-9]+)/i) { $filterOp = $1; } + if ($_ =~ /conn= *([0-9A-Z]+)/i) { $filterConn = $1; } + if ($_ =~ /op= *([0-9\-]+)/i) { $filterOp = $1; } } $filterCount++; if($usage =~ /u/ || $verb eq "yes"){ @@ -2157,11 +2176,11 @@ sub parseLineNormal if ($_ =~ /scope= *([0-9]+)/i) { $scopeVal = $1; } - if ($_ =~ /conn= *([0-9]+)/i) { + if ($_ =~ /conn= *([0-9A-Z]+)/i) { $baseConn = $1; $scopeConn = $1; } - if ($_ =~ /op= *([0-9]+)/i) { + if ($_ =~ /op= *([0-9\-]+)/i) { $baseOp = $1; $scopeOp = $1; } @@ -2170,9 +2189,9 @@ sub parseLineNormal push @{$arrays->{baseval}}, $tmpp; push @{$arrays->{baseconn}}, $baseConn; push @{$arrays->{baseop}}, $baseOp; - push @{$arrays->{scopeval}}, $scopeVal; - push @{$arrays->{scopeconn}}, $scopeConn; - push @{$arrays->{scopeop}}, $scopeOp; + push @{$arrays->{scopeval}}, $scopeTxt[$scopeVal]; + push @{$arrays->{scopeconn}}, $scopeConn; + push @{$arrays->{scopeop}}, $scopeOp; } $baseCount++; $scopeCount++; @@ -2191,12 +2210,12 @@ sub parseLineNormal $ds6x = "true"; $badPwdCount++; } elsif (/ err=49 tag=/ ){ - if ($_ =~ /conn= *([0-9]+)/i ){ + if ($_ =~ /conn= *([0-9A-Z]+)/i ){ push @{$arrays->{badpwdconn}}, $1; $ip = getIPfromConn($1); $badPwdCount++; } - if ($_ =~ /op= *([0-9]+)/i ){ + if ($_ =~ /op= *([0-9\-]+)/i ){ push @{$arrays->{badpwdop}}, $1; } push @{$arrays->{badpwdip}}, $ip; commit 34468104d63f6d439ff2ed8bc0b9afa2751b5b0a Author: Rich Megginson <rmeggins(a)redhat.com&gt; Date: Tue Jul 2 14:38:49 2013 -0600 Ticket #47387 - improve logconv.pl performance with large access logs https://fedorahosted.org/389/ticket/47387 Reviewed by: mreynolds (Thanks!) Branch: master Fix Description: Fix some additional uninitialized variable warnings Platforms tested: RHEL6 x86_64 Flag Day: no Doc impact: no (cherry picked from commit 8a23f5e0af41291d28c4ce7251d9bb2e94a5994c) (cherry picked from commit 7a107bdb67dad58728977b3f410df1d94c6cdb91) diff --git a/ldap/admin/src/logconv.pl b/ldap/admin/src/logconv.pl index b628d03..ec0fdf5 100755 --- a/ldap/admin/src/logconv.pl +++ b/ldap/admin/src/logconv.pl @@ -673,7 +673,7 @@ if ($verb eq "yes" || $usage =~ /u/){ my $notesCount = "1"; my $unindexedIp; - for (my $n = 0; $n <= scalar(@{$notesEtime}); $n++){ + for (my $n = 0; $n < scalar(@{$notesEtime}); $n++){ if($conn_hash->{$notesConn->[$n]} eq ""){ $unindexedIp = "?"; } else { @@ -726,7 +726,7 @@ if ($verb eq "yes" || $usage =~ /u/){ my $notesCount = "1"; my $unindexedIp; - for (my $n = 0; $n <= scalar(@{$notesEtime}); $n++){ + for (my $n = 0; $n < scalar(@{$notesEtime}); $n++){ if($conn_hash->{$notesConn->[$n]} eq ""){ $unindexedIp = "?"; } else { @@ -1821,7 +1821,7 @@ sub parseLineNormal $con = $1; if ($_ =~ /op= *([0-9]+)/i){ $op = $1;} } - for (my $i=0; $i <= $vlvCount;$i++){ + for (my $i=0; $i < $vlvCount;$i++){ if ($vlvconn[$i] eq $con && $vlvop[$i] eq $op){ $vlvNotesACount++; $isVlvNotes="1";} } if($isVlvNotes == 0){ @@ -1846,7 +1846,7 @@ sub parseLineNormal $con = $1; if ($_ =~ /op= *([0-9]+)/i){ $op = $1;} } - for (my $i=0; $i <= $vlvCount;$i++){ + for (my $i=0; $i < $vlvCount;$i++){ if ($vlvconn[$i] eq $con && $vlvop[$i] eq $op){ $vlvNotesUCount++; $isVlvNotes="1";} } if($isVlvNotes == 0){ commit 481f2e3aec2834f342c01919d91eb6cfd80f6ca1 Author: Rich Megginson <rmeggins(a)redhat.com&gt; Date: Mon Jun 10 20:04:20 2013 -0600 Ticket #47387 - improve logconv.pl performance with large access logs https://fedorahosted.org/389/ticket/47387 Reviewed by: mreynolds (Thanks!) Branch: master Fix Description: The primary fix is to use tied hashes and arrays backed by DB_File database files. Hashes use DB_HASH and arrays use DB_RECNO. These fixes cut down the time by a factor of 10 or more, and considerably reduce the size of the disk files. There is still room for optimization as given by the Devel::NYTProf package, but this should at least give us acceptable performance. I also did some perl "linting" - running with use warnings and use strict - which revealed a lot of code which was cleaned up. Platforms tested: RHEL6 x86_64 Flag Day: no Doc impact: no (cherry picked from commit 8e35cc8b418bc46fbeb28f4378d24f9c2cab0353) diff --git a/ldap/admin/src/logconv.pl b/ldap/admin/src/logconv.pl index 8ae72da..b628d03 100755 --- a/ldap/admin/src/logconv.pl +++ b/ldap/admin/src/logconv.pl @@ -43,9 +43,14 @@ # # Check for usage # +use strict; +use warnings; +use warnings 'untie'; use Time::Local; use IO::File; use Getopt::Long; +use DB_File; +use sigtrap qw(die normal-signals); Getopt::Long::Configure ("bundling"); Getopt::Long::Configure ("permute"); @@ -60,19 +65,43 @@ if ($#ARGV < 0){; # # ####################################### -$file_count = 0; -$arg_count = 0; -$logversion = "7.0"; -$sizeCount = "20"; -$startFlag = 0; -$startTime = 0; -$endFlag = 0; -$endTime = 0; -$reportStats = ""; -$dataLocation = "/tmp"; -$startTLSoid = "1.3.6.1.4.1.1466.20037"; -$s_stats = new_stats_block( ); -$m_stats = new_stats_block( ); +my $file_count = 0; +my $arg_count = 0; +my $logversion = "7.0"; +my $sizeCount = "20"; +my $startFlag = 0; +my $startTime = 0; +my $endFlag = 0; +my $endTime = 0; +my $reportStats = ""; +my $dataLocation = "/tmp"; +my $startTLSoid = "1.3.6.1.4.1.1466.20037"; +my $s_stats = new_stats_block( ); +my $m_stats = new_stats_block( ); +my $verb = "no"; +my @excludeIP; +my $xi = 0; +my $bindReportDN; +my $usage = ""; +my @latency; +my @openConnection; +my @errorCode; +my @errtext; +my @errornum; +my @errornum2; +my $ds6x = "false"; +my $connCodeCount = 0; +my %connList; +my %bindReport; +my @vlvconn; +my @vlvop; +my @start_time_of_connection; +my @end_time_of_connection; +my @fds; +my $fdds = 0; +my $reportBinds = "no"; +my $rootDN = ""; +my $needCleanup = 0; GetOptions( 'd|rootDN=s' => \$rootDN, @@ -129,6 +158,7 @@ if($rootDN eq ""){ # # get the logs # +my @files = (); while($arg_count <= $#ARGV){ $files[$file_count] = $ARGV[$arg_count]; $file_count++; @@ -155,127 +185,93 @@ if ($sizeCount eq "all"){$sizeCount = "100000";} print "\nAccess Log Analyzer $logversion\n"; print "\nCommand: logconv.pl @ARGV\n\n"; -$rootDNBindCount = 0; -$anonymousBindCount = 0; -$unindexedSrchCountNotesA = 0; -$unindexedSrchCountNotesU = 0; -$vlvNotesACount= 0; -$vlvNotesUCount= 0; -$srchCount = 0; -$fdTaken = 0; -$fdReturned = 0; -$highestFdTaken = 0; -$unbindCount = 0; -$cmpCount = 0; -$modCount = 0; -$delCount = 0; -$addCount = 0; -$modrdnCount = 0; -$abandonCount = 0; -$extopCount = 0; -$vlvCount = 0; -$errorCount = 0; -$proxiedAuthCount = 0; -$serverRestartCount = 0; -$resourceUnavailCount = 0; -$brokenPipeCount = 0; -$v2BindCount = 0; -$v3BindCount = 0; -$vlvSortCount = 0; -$connResetByPeerCount = 0; -$isVlvNotes = 0; -$successCount = 0; -$sslCount = 0; -$sslClientBindCount = 0; -$sslClientFailedCount = 0; -$objectclassTopCount= 0; -$pagedSearchCount = 0; -$bindCount = 0; -$filterCount = 0; -$baseCount = 0; -$scopeCount = 0; -$allOps = 0; -$allResults = 0; -$badPwdCount = 0; -$saslBindCount = 0; -$internalOpCount = 0; -$entryOpCount = 0; -$referralCount = 0; -$anyAttrs = 0; -$persistentSrchCount = 0; -$maxBerSizeCount = 0; -$connectionCount = 0; -$timerange = 0; -$simConnection = 0; -$maxsimConnection = 0; -$firstFile = 1; -$elapsedDays = 0; -$logCount = 0; -$startTLSCount = 0; -$ldapiCount = 0; -$autobindCount = 0; -$limit = 25000; # number of lines processed to trigger output - -# hash files -$ATTR = "$dataLocation/attr.logconv"; -$RC = "$dataLocation/rc.logconv"; -$SRC = "$dataLocation/src.logconv"; -$RSRC = "$dataLocation/rsrc.logconv"; -$EXCOUNT = "$dataLocation/excount.logconv"; -$CONN_HASH = "$dataLocation/conn_hash.logconv"; -$IP_HASH = "$dataLocation/ip_hash.logconv"; -$CONNCOUNT = "$dataLocation/conncount.logconv"; -$NENTRIES = "$dataLocation/nentries.logconv"; -$FILTER = "$dataLocation/filter.logconv"; -$BASE = "$dataLocation/base.logconv"; -$DS6XBADPWD = "$dataLocation/ds6xbadpwd.logconv"; -$SASLMECH = "$dataLocation/saslmech.logconv"; -$BINDLIST = "$dataLocation/bindlist.logconv"; -$ETIME = "$dataLocation/etime.logconv"; -$OID = "$dataLocation/oid.logconv"; - -# array files -$SRCH_CONN = "$dataLocation/srchconn.logconv"; -$SRCH_OP = "$dataLocation/srchop.logconv"; -$DEL_CONN = "$dataLocation/delconn.logconv"; -$DEL_OP = "$dataLocation/delop.logconv"; -$MOD_CONN = "$dataLocation/modconn.logconv"; -$MOD_OP = "$dataLocation/modop.logconv"; -$ADD_CONN = "$dataLocation/addconn.logconv"; -$ADD_OP = "$dataLocation/addop.logconv"; -$MODRDN_CONN = "$dataLocation/modrdnconn.logconv"; -$MODRDN_OP = "$dataLocation/modrdnop.logconv"; -$CMP_CONN = "$dataLocation/cmpconn.logconv"; -$CMP_OP = "$dataLocation/cmpop.logconv"; -$TARGET_CONN = "$dataLocation/targetconn.logconv"; -$TARGET_OP = "$dataLocation/targetop.logconv"; -$MSGID = "$dataLocation/msgid.logconv"; -$BIND_CONN = "$dataLocation/bindconn.logconv"; -$BIND_OP = "$dataLocation/bindop.logconv"; -$UNBIND_CONN = "$dataLocation/unbindconn.logconv"; -$UNBIND_OP = "$dataLocation/unbindop.logconv"; -$EXT_CONN = "$dataLocation/extconn.logconv"; -$EXT_OP = "$dataLocation/extop.logconv"; -$NOTES_A_ETIME = "$dataLocation/notesAetime.logconv"; -$NOTES_A_CONN = "$dataLocation/notesAconn.logconv"; -$NOTES_A_OP = "$dataLocation/notesAop.logconv"; -$NOTES_A_TIME = "$dataLocation/notesAtime.logconv"; -$NOTES_A_NENTRIES = "$dataLocation/notesAnentries.logconv"; -$NOTES_U_ETIME = "$dataLocation/notesUetime.logconv"; -$NOTES_U_CONN = "$dataLocation/notesUconn.logconv"; -$NOTES_U_OP = "$dataLocation/notesUop.logconv"; -$NOTES_U_TIME = "$dataLocation/notesUtime.logconv"; -$NOTES_U_NENTRIES = "$dataLocation/notesUnentries.logconv"; -$BADPWDCONN = "$dataLocation/badpwdconn.logconv"; -$BADPWDOP = "$dataLocation/badpwdop.logconv"; -$BADPWDIP = "$dataLocation/badpwdip.logconv"; - -# info files -$BINDINFO = "$dataLocation/bindinfo.logconv"; -$BASEINFO = "$dataLocation/baseinfo.logconv"; -$FILTERINFO = "$dataLocation/filterinfo.logconv"; -$SCOPEINFO = "$dataLocation/scopeinfo.logconv"; - +my $rootDNBindCount = 0; +my $anonymousBindCount = 0; +my $unindexedSrchCountNotesA = 0; +my $unindexedSrchCountNotesU = 0; +my $vlvNotesACount= 0; +my $vlvNotesUCount= 0; +my $srchCount = 0; +my $fdTaken = 0; +my $fdReturned = 0; +my $highestFdTaken = 0; +my $unbindCount = 0; +my $cmpCount = 0; +my $modCount = 0; +my $delCount = 0; +my $addCount = 0; +my $modrdnCount = 0; +my $abandonCount = 0; +my $extopCount = 0; +my $vlvCount = 0; +my $errorCount = 0; +my $proxiedAuthCount = 0; +my $serverRestartCount = 0; +my $resourceUnavailCount = 0; +my $brokenPipeCount = 0; +my $v2BindCount = 0; +my $v3BindCount = 0; +my $vlvSortCount = 0; +my $connResetByPeerCount = 0; +my $isVlvNotes = 0; +my $successCount = 0; +my $sslCount = 0; +my $sslClientBindCount = 0; +my $sslClientFailedCount = 0; +my $objectclassTopCount= 0; +my $pagedSearchCount = 0; +my $bindCount = 0; +my $filterCount = 0; +my $baseCount = 0; +my $scopeCount = 0; +my $allOps = 0; +my $allResults = 0; +my $badPwdCount = 0; +my $saslBindCount = 0; +my $internalOpCount = 0; +my $entryOpCount = 0; +my $referralCount = 0; +my $anyAttrs = 0; +my $persistentSrchCount = 0; +my $maxBerSizeCount = 0; +my $connectionCount = 0; +my $timerange = 0; +my $simConnection = 0; +my $maxsimConnection = 0; +my $firstFile = 1; +my $elapsedDays = 0; +my $logCount = 0; +my $startTLSCount = 0; +my $ldapiCount = 0; +my $autobindCount = 0; +my $limit = 25000; # number of lines processed to trigger output + +my @removefiles = (); + +my @conncodes = qw(A1 B1 B4 T1 T2 B2 B3 R1 P1 P2 U1); +my %conn = (); +map {$conn{$_} = $_} @conncodes; + +# hash db-backed hashes +my @hashnames = qw(attr rc src rsrc excount conn_hash ip_hash conncount nentries + filter base ds6xbadpwd saslmech bindlist etime oid); +# need per connection code ip address counts - so use a hash table +# for each connection code - key is ip, val is count +push @hashnames, @conncodes; +my $hashes = openHashFiles($dataLocation, @hashnames); + +# recno db-backed arrays/lists +my @arraynames = qw(srchconn srchop delconn delop modconn modop addconn addop modrdnconn modrdnop + cmpconn cmpop targetconn targetop msgid bindconn bindop binddn unbindconn unbindop + extconn extop notesAetime notesAconn notesAop notesAtime notesAnentries + notesUetime notesUconn notesUop notesUtime notesUnentries badpwdconn + badpwdop badpwdip baseval baseconn baseop scopeval scopeconn scopeop + filterval filterconn filterop); +my $arrays = openArrayFiles($dataLocation, @arraynames); + +$needCleanup = 1; + +my @err; $err[0] = "Successful Operations\n"; $err[1] = "Operations Error(s)\n"; $err[2] = "Protocal Errors\n"; @@ -338,19 +334,7 @@ $err[95] = "More Results To Return\n"; $err[96] = "Client Loop\n"; $err[97] = "Referral Limit Exceeded\n"; - -$conn{"A1"} = "A1"; -$conn{"B1"} = "B1"; -$conn{"B4"} = "B4"; -$conn{"T1"} = "T1"; -$conn{"T2"} = "T2"; -$conn{"B2"} = "B2"; -$conn{"B3"} = "B3"; -$conn{"R1"} = "R1"; -$conn{"P1"} = "P1"; -$conn{"P2"} = "P2"; -$conn{"U1"} = "U1"; - +my %connmsg; $connmsg{"A1"} = "Client Aborted Connections"; $connmsg{"B1"} = "Bad Ber Tag Encountered"; $connmsg{"B4"} = "Server failed to flush data (response) back to Client"; @@ -363,7 +347,7 @@ $connmsg{"P1"} = "Plugin"; $connmsg{"P2"} = "Poll"; $connmsg{"U1"} = "Cleanly Closed Connections"; -%monthname = ( +my %monthname = ( "Jan" => 0, "Feb" => 1, "Mar" => 2, @@ -379,7 +363,17 @@ $connmsg{"U1"} = "Cleanly Closed Connections"; ); -openDataFiles(); +my $linesProcessed; +my $lineBlockCount; +my $cursize = 0; +sub statusreport { + if ($lineBlockCount > $limit) { + my $curpos = tell(LOG); + my $percent = $curpos/$cursize*100.0; + print sprintf "%10d Lines Processed %12d of %12d bytes (%.3f%%)\n",--$linesProcessed,$curpos,$cursize,$percent; + $lineBlockCount = 0; + } +} ########################################## # # @@ -394,6 +388,7 @@ print "Processing $file_count Access Log(s)...\n\n"; #print "Filename\t\t\t Total Lines\n"; #print "--------------------------------------------------\n"; +my $skipFirstFile = 0; if ($file_count > 1 && $files[0] =~ /\/access$/){ $files[$file_count] = $files[0]; $file_count++; @@ -401,25 +396,30 @@ if ($file_count > 1 && $files[0] =~ /\/access$/){ } $logCount = $file_count; -for ($count=0; $count < $file_count; $count++){ +my $logline; +my $totalLineCount = 0; + +for (my $count=0; $count < $file_count; $count++){ # we moved access to the end of the list, so if its the first file skip it if($file_count > 1 && $count == 0 && $skipFirstFile == 1){ next; } - $logsize = `wc -l $files[$count]`; - $logsize =~ /([0-9]+)/; $linesProcessed = 0; $lineBlockCount = 0; $logCount--; + my $logCountStr; if($logCount < 10 ){ # add a zero for formatting purposes $logCountStr = "0" . $logCount; } else { $logCountStr = $logCount; } - print sprintf "[%s] %-30s\tlines: %7s\n",$logCountStr, $files[$count], $1; + my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$atime,$mtime,$ctime,$blksize,$blocks); + ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$cursize, + $atime,$mtime,$ctime,$blksize,$blocks) = stat($files[$count]); + print sprintf "[%s] %-30s\tsize (bytes): %12s\n",$logCountStr, $files[$count], $cursize; open(LOG,"$files[$count]") or do { openFailed($!, $files[$count]) }; - $firstline = "yes"; + my $firstline = "yes"; while(<LOG>){ unless ($endFlag) { if ($firstline eq "yes"){ @@ -442,7 +442,7 @@ for ($count=0; $count < $file_count; $count++){ print_stats_block( $s_stats ); print_stats_block( $m_stats ); $totalLineCount = $totalLineCount + $linesProcessed; - if($linesProcessed => $limit){print sprintf " %10s Lines Processed\n\n",--$linesProcessed;} + statusreport(); } print "\n\nTotal Log Lines Analysed: " . ($totalLineCount - 1) . "\n"; @@ -457,9 +457,11 @@ $allOps = $srchCount + $modCount + $addCount + $cmpCount + $delCount + $modrdnCo # if we are using startTime & endTime then we need to clean it up for our processing +my $start; if($startTime){ if ($start =~ / *([0-9a-z:\/]+)/i){$start=$1;} } +my $end; if($endTime){ if ($end =~ / *([0-9a-z:\/]+)/i){$end =$1;} } @@ -468,8 +470,10 @@ if($endTime){ # Get the start time in seconds # -$logStart = $start; - +my $logStart = $start; +my $logDate; +my @dateComps; +my ($timeMonth, $timeDay, $timeYear, $dateTotal); if ($logStart =~ / *([0-9A-Z\/]+)/i ){ $logDate = $1; @dateComps = split /\//, $logDate; @@ -481,6 +485,9 @@ if ($logStart =~ / *([0-9A-Z\/]+)/i ){ $dateTotal = $timeMonth + $timeDay + $timeYear; } +my $logTime; +my @timeComps; +my ($timeHour, $timeMinute, $timeSecond, $timeTotal); if ($logStart =~ / *(:[0-9:]+)/i ){ $logTime = $1; @timeComps = split /:/, $logTime; @@ -491,14 +498,14 @@ if ($logStart =~ / *(:[0-9:]+)/i ){ $timeTotal = $timeHour + $timeMinute + $timeSecond; } -$startTotal = $timeTotal + $dateTotal; +my $startTotal = $timeTotal + $dateTotal; # # Get the end time in seconds # -$logEnd = $end; - +my $logEnd = $end; +my ($endDay, $endMonth, $endYear, $endTotal); if ($logEnd =~ / *([0-9A-Z\/]+)/i ){ $logDate = $1; @dateComps = split /\//, $logDate; @@ -506,10 +513,11 @@ if ($logEnd =~ / *([0-9A-Z\/]+)/i ){ $endDay = $dateComps[0] *3600 * 24; $endMonth = 1 + $monthname{$dateComps[1]}; $endMonth = $endMonth * 3600 * 24 * 30; - $endYear = $endTotal + $dateComps[2] *365 * 3600 * 24 ; + $endYear = $dateComps[2] *365 * 3600 * 24 ; $dateTotal = $endDay + $endMonth + $endYear; } +my ($endHour, $endMinute, $endSecond); if ($logEnd =~ / *(:[0-9:]+)/i ){ $logTime = $1; @timeComps = split /:/, $logTime; @@ -525,8 +533,8 @@ $endTotal = $timeTotal + $dateTotal; # # Tally the numbers # -$totalTimeInSecs = $endTotal - $startTotal; -$remainingTimeInSecs = $totalTimeInSecs; +my $totalTimeInSecs = $endTotal - $startTotal; +my $remainingTimeInSecs = $totalTimeInSecs; # # Calculate the elapsed time @@ -540,33 +548,21 @@ while(($remainingTimeInSecs - 86400) > 0){ } # hours +my $elapsedHours = 0; while(($remainingTimeInSecs - 3600) > 0){ $elapsedHours++; $remainingTimeInSecs = $remainingTimeInSecs - 3600; } # minutes +my $elapsedMinutes = 0; while($remainingTimeInSecs - 60 > 0){ $elapsedMinutes++; $remainingTimeInSecs = $remainingTimeInSecs - 60; } # seconds -$elapsedSeconds = $remainingTimeInSecs; - -# Initialize empty values -if($elapsedHours eq ""){ - $elapsedHours = "0"; -} -if($elapsedMinutes eq ""){ - $elapsedMinutes = "0"; -} -if($elapsedSeconds eq ""){ - $elapsedSeconds = "0"; -} - -&closeDataFiles(); - +my $elapsedSeconds = $remainingTimeInSecs; ##################################### # # @@ -605,20 +601,21 @@ print " - LDAPI Connections: $ldapiCount\n"; print "Peak Concurrent Connections: $maxsimConnection\n"; print "Total Operations: $allOps\n"; print "Total Results: $allResults\n"; +my ($perf, $tmp); if ($allOps ne "0"){ - print sprintf "Overall Performance: %.1f%\n\n" , ($perf = ($tmp = ($allResults / $allOps)*100) > 100 ? 100.0 : $tmp) ; + print sprintf "Overall Performance: %.1f%%\n\n" , ($perf = ($tmp = ($allResults / $allOps)*100) > 100 ? 100.0 : $tmp) ; } else { print "Overall Performance: No Operations to evaluate\n\n"; } -$searchStat = sprintf "(%.2f/sec) (%.2f/min)\n",($srchCount / $totalTimeInSecs), $srchCount / ($totalTimeInSecs/60); -$modStat = sprintf "(%.2f/sec) (%.2f/min)\n",$modCount / $totalTimeInSecs, $modCount/($totalTimeInSecs/60); -$addStat = sprintf "(%.2f/sec) (%.2f/min)\n",$addCount/$totalTimeInSecs, $addCount/($totalTimeInSecs/60); -$deleteStat = sprintf "(%.2f/sec) (%.2f/min)\n",$delCount/$totalTimeInSecs, $delCount/($totalTimeInSecs/60); -$modrdnStat = sprintf "(%.2f/sec) (%.2f/min)\n",$modrdnCount/$totalTimeInSecs, $modrdnCount/($totalTimeInSecs/60); -$compareStat = sprintf "(%.2f/sec) (%.2f/min)\n",$cmpCount/$totalTimeInSecs, $cmpCount/($totalTimeInSecs/60); -$bindCountStat = sprintf "(%.2f/sec) (%.2f/min)\n",$bindCount/$totalTimeInSecs, $bindCount/($totalTimeInSecs/60); +my $searchStat = sprintf "(%.2f/sec) (%.2f/min)\n",($srchCount / $totalTimeInSecs), $srchCount / ($totalTimeInSecs/60); +my $modStat = sprintf "(%.2f/sec) (%.2f/min)\n",$modCount / $totalTimeInSecs, $modCount/($totalTimeInSecs/60); +my $addStat = sprintf "(%.2f/sec) (%.2f/min)\n",$addCount/$totalTimeInSecs, $addCount/($totalTimeInSecs/60); +my $deleteStat = sprintf "(%.2f/sec) (%.2f/min)\n",$delCount/$totalTimeInSecs, $delCount/($totalTimeInSecs/60); +my $modrdnStat = sprintf "(%.2f/sec) (%.2f/min)\n",$modrdnCount/$totalTimeInSecs, $modrdnCount/($totalTimeInSecs/60); +my $compareStat = sprintf "(%.2f/sec) (%.2f/min)\n",$cmpCount/$totalTimeInSecs, $cmpCount/($totalTimeInSecs/60); +my $bindCountStat = sprintf "(%.2f/sec) (%.2f/min)\n",$bindCount/$totalTimeInSecs, $bindCount/($totalTimeInSecs/60); format STDOUT = Searches: @<<<<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< @@ -658,148 +655,110 @@ print "Unindexed Searches: $unindexedSrchCountNotesA\n"; print "Unindexed Components: $unindexedSrchCountNotesU\n"; if ($verb eq "yes" || $usage =~ /u/){ if ($unindexedSrchCountNotesA > 0){ - %conn_hash = getHashFromFile($CONN_HASH); - @notesConn = getArrayFromFile($NOTES_A_CONN); - @notesOp = getArrayFromFile($NOTES_A_OP); - @notesEtime = getArrayFromFile($NOTES_A_ETIME); - @notesTime = getArrayFromFile($NOTES_A_TIME); - @notesNentries = getArrayFromFile($NOTES_A_NENTRIES); - getInfoArraysFromFile($BASEINFO); - @base_val = @fileArray1; - @base_conn = @fileArray2; - @base_op = @fileArray3; - getInfoArraysFromFile($SCOPEINFO); - @scope_val = @fileArray1; - @scope_conn = @fileArray2; - @scope_op = @fileArray3; - getInfoArraysFromFile($FILTERINFO); - @filter_val = @fileArray1; - @filter_conn = @fileArray2; - @filter_op = @fileArray3; - - $notesCount = "1"; - for ($n = 0; $n <= $#notesEtime; $n++){ - @alreadyseenDN = (); - if($conn_hash{$notesConn[$n]} eq ""){ + my $conn_hash = $hashes->{conn_hash}; + my $notesConn = $arrays->{notesAconn}; + my $notesOp = $arrays->{notesAop}; + my $notesEtime = $arrays->{notesAetime}; + my $notesTime = $arrays->{notesAtime}; + my $notesNentries = $arrays->{notesAnentries}; + my $base_val = $arrays->{baseval}; + my $base_conn = $arrays->{baseconn}; + my $base_op = $arrays->{baseop}; + my $scope_val = $arrays->{scopeval}; + my $scope_conn = $arrays->{scopeconn}; + my $scope_op = $arrays->{scopeop}; + my $filter_val = $arrays->{filterval}; + my $filter_conn = $arrays->{filterconn}; + my $filter_op = $arrays->{filterop}; + + my $notesCount = "1"; + my $unindexedIp; + for (my $n = 0; $n <= scalar(@{$notesEtime}); $n++){ + if($conn_hash->{$notesConn->[$n]} eq ""){ $unindexedIp = "?"; } else { - $unindexedIp = $conn_hash{$notesConn[$n]}; + $unindexedIp = $conn_hash->{$notesConn->[$n]}; } print "\n Unindexed Search #".$notesCount."\n"; $notesCount++; - print " - Date/Time: $notesTime[$n]\n"; - print " - Connection Number: $notesConn[$n]\n"; - print " - Operation Number: $notesOp[$n]\n"; - print " - Etime: $notesEtime[$n]\n"; - print " - Nentries: $notesNentries[$n]\n"; + print " - Date/Time: $notesTime->[$n]\n"; + print " - Connection Number: $notesConn->[$n]\n"; + print " - Operation Number: $notesOp->[$n]\n"; + print " - Etime: $notesEtime->[$n]\n"; + print " - Nentries: $notesNentries->[$n]\n"; print " - IP Address: $unindexedIp\n"; - for ($nnn = 0; $nnn < $baseCount; $nnn++){ - if ($notesConn[$n] eq $base_conn[$nnn] && $notesOp[$n] eq $base_op[$nnn]){ - print " - Search Base: $base_val[$nnn]\n"; + for (my $nnn = 0; $nnn < $baseCount; $nnn++){ + if ($notesConn->[$n] eq $base_conn->[$nnn] && $notesOp->[$n] eq $base_op->[$nnn]){ + print " - Search Base: $base_val->[$nnn]\n"; last; } } - for ($nnn = 0; $nnn < $scopeCount; $nnn++){ - if ($notesConn[$n] eq $scope_conn[$nnn] && $notesOp[$n] eq $scope_op[$nnn]){ - print " - Search Scope: $scope_val[$nnn]\n"; + for (my $nnn = 0; $nnn < $scopeCount; $nnn++){ + if ($notesConn->[$n] eq $scope_conn->[$nnn] && $notesOp->[$n] eq $scope_op->[$nnn]){ + print " - Search Scope: $scope_val->[$nnn]\n"; last; } } - for ($nnn = 0; $nnn < $filterCount; $nnn++){ - if ($notesConn[$n] eq $filter_conn[$nnn] && $notesOp[$n] eq $filter_op[$nnn]){ - print " - Search Filter: $filter_val[$nnn]\n"; + for (my $nnn = 0; $nnn < $filterCount; $nnn++){ + if ($notesConn->[$n] eq $filter_conn->[$nnn] && $notesOp->[$n] eq $filter_op->[$nnn]){ + print " - Search Filter: $filter_val->[$nnn]\n"; last; } } } - undef %conn_hash; - undef @notesConn; - undef @notesOp; - undef @notesEtime; - undef @notesTime; - undef @notesNentries; - undef @notesIp; - undef @filter_val; - undef @filter_conn; - undef @filter_op; - undef @base_val; - undef @base_conn; - undef @base_op; - undef @scope_val; - undef @scope_conn; - undef @scope_op; } if ($unindexedSrchCountNotesU > 0){ - %conn_hash = getHashFromFile($CONN_HASH); - @notesConn = getArrayFromFile($NOTES_U_CONN); - @notesOp = getArrayFromFile($NOTES_U_OP); - @notesEtime = getArrayFromFile($NOTES_U_ETIME); - @notesTime = getArrayFromFile($NOTES_U_TIME); - @notesNentries = getArrayFromFile($NOTES_U_NENTRIES); - getInfoArraysFromFile($BASEINFO); - @base_val = @fileArray1; - @base_conn = @fileArray2; - @base_op = @fileArray3; - getInfoArraysFromFile($SCOPEINFO); - @scope_val = @fileArray1; - @scope_conn = @fileArray2; - @scope_op = @fileArray3; - getInfoArraysFromFile($FILTERINFO); - @filter_val = @fileArray1; - @filter_conn = @fileArray2; - @filter_op = @fileArray3; - - $notesCount = "1"; - for ($n = 0; $n <= $#notesEtime; $n++){ - @alreadyseenDN = (); - if($conn_hash{$notesConn[$n]} eq ""){ + my $conn_hash = $hashes->{conn_hash}; + my $notesConn = $arrays->{notesUconn}; + my $notesOp = $arrays->{notesUop}; + my $notesEtime = $arrays->{notesUetime}; + my $notesTime = $arrays->{notesUtime}; + my $notesNentries = $arrays->{notesUnentries}; + my $base_val = $arrays->{baseval}; + my $base_conn = $arrays->{baseconn}; + my $base_op = $arrays->{baseop}; + my $scope_val = $arrays->{scopeval}; + my $scope_conn = $arrays->{scopeconn}; + my $scope_op = $arrays->{scopeop}; + my $filter_val = $arrays->{filterval}; + my $filter_conn = $arrays->{filterconn}; + my $filter_op = $arrays->{filterop}; + + my $notesCount = "1"; + my $unindexedIp; + for (my $n = 0; $n <= scalar(@{$notesEtime}); $n++){ + if($conn_hash->{$notesConn->[$n]} eq ""){ $unindexedIp = "?"; } else { - $unindexedIp = $conn_hash{$notesConn[$n]}; + $unindexedIp = $conn_hash->{$notesConn->[$n]}; } print "\n Unindexed Components #".$notesCount."\n"; $notesCount++; - print " - Date/Time: $notesTime[$n]\n"; - print " - Connection Number: $notesConn[$n]\n"; - print " - Operation Number: $notesOp[$n]\n"; - print " - Etime: $notesEtime[$n]\n"; - print " - Nentries: $notesNentries[$n]\n"; + print " - Date/Time: $notesTime->[$n]\n"; + print " - Connection Number: $notesConn->[$n]\n"; + print " - Operation Number: $notesOp->[$n]\n"; + print " - Etime: $notesEtime->[$n]\n"; + print " - Nentries: $notesNentries->[$n]\n"; print " - IP Address: $unindexedIp\n"; - for ($nnn = 0; $nnn < $baseCount; $nnn++){ - if ($notesConn[$n] eq $base_conn[$nnn] && $notesOp[$n] eq $base_op[$nnn]){ - print " - Search Base: $base_val[$nnn]\n"; + for (my $nnn = 0; $nnn < $baseCount; $nnn++){ + if ($notesConn->[$n] eq $base_conn->[$nnn] && $notesOp->[$n] eq $base_op->[$nnn]){ + print " - Search Base: $base_val->[$nnn]\n"; last; } } - for ($nnn = 0; $nnn < $scopeCount; $nnn++){ - if ($notesConn[$n] eq $scope_conn[$nnn] && $notesOp[$n] eq $scope_op[$nnn]){ - print " - Search Scope: $scope_val[$nnn]\n"; + for (my $nnn = 0; $nnn < $scopeCount; $nnn++){ + if ($notesConn->[$n] eq $scope_conn->[$nnn] && $notesOp->[$n] eq $scope_op->[$nnn]){ + print " - Search Scope: $scope_val->[$nnn]\n"; last; } } - for ($nnn = 0; $nnn < $filterCount; $nnn++){ - if ($notesConn[$n] eq $filter_conn[$nnn] && $notesOp[$n] eq $filter_op[$nnn]){ - print " - Search Filter: $filter_val[$nnn]\n"; + for (my $nnn = 0; $nnn < $filterCount; $nnn++){ + if ($notesConn->[$n] eq $filter_conn->[$nnn] && $notesOp->[$n] eq $filter_op->[$nnn]){ + print " - Search Filter: $filter_val->[$nnn]\n"; last; } } } - undef %conn_hash; - undef @notesConn; - undef @notesOp; - undef @notesEtime; - undef @notesTime; - undef @notesNentries; - undef @notesIp; - undef @filter_val; - undef @filter_conn; - undef @filter_op; - undef @base_val; - undef @base_conn; - undef @base_op; - undef @scope_val; - undef @scope_conn; - undef @scope_op; } } # end of unindexed search report @@ -809,10 +768,12 @@ print "FDs Returned: $fdReturned\n"; print "Highest FD Taken: $highestFdTaken\n\n"; print "Broken Pipes: $brokenPipeCount\n"; if ($brokenPipeCount > 0){ - foreach $key (sort { $rc{$b} <=> $rc{$a} } keys %rc) { - if ($rc{$key} > 0){ + my $rc = $hashes->{rc}; + my @etext; + foreach my $key (sort { $rc->{$b} <=> $rc->{$a} } keys %{$rc}) { + if ($rc->{$key} > 0){ if ($conn{$key} eq ""){$conn{$key} = "**Unknown**";} - push @etext, sprintf " - %-4s (%2s) %-40s\n",$rc{$key},$conn{$key},$connmsg{$key + push @etext, sprintf " - %-4s (%2s) %-40s\n",$rc->{$key},$conn{$key},$connmsg{$key }; } } @@ -822,10 +783,12 @@ if ($brokenPipeCount > 0){ print "Connections Reset By Peer: $connResetByPeerCount\n"; if ($connResetByPeerCount > 0){ - foreach $key (sort { $src{$b} <=> $src{$a} } keys %src) { - if ($src{$key} > 0){ + my $src = $hashes->{src}; + my @retext; + foreach my $key (sort { $src->{$b} <=> $src->{$a} } keys %{$src}) { + if ($src->{$key} > 0){ if ($conn{$key} eq ""){$conn{$key} = "**Unknown**";} - push @retext, sprintf " - %-4s (%2s) %-40s\n",$src{$key},$conn{$key},$connmsg{$key + push @retext, sprintf " - %-4s (%2s) %-40s\n",$src->{$key},$conn{$key},$connmsg{$key }; } } @@ -835,10 +798,12 @@ if ($connResetByPeerCount > 0){ print "Resource Unavailable: $resourceUnavailCount\n"; if ($resourceUnavailCount > 0){ - foreach $key (sort { $rsrc{$b} <=> $rsrc{$a} } keys %rsrc) { - if ($rsrc{$key} > 0){ + my $rsrc = $hashes->{rsrc}; + my @rtext; + foreach my $key (sort { $rsrc->{$b} <=> $rsrc->{$a} } keys %{$rsrc}) { + if ($rsrc->{$key} > 0){ if ($conn{$key} eq ""){$conn{$key} = "**Resource Issue**";} - push @rtext, sprintf " - %-4s (%2s) %-40s\n",$rsrc{$key},$conn{$key},$connmsg{$key}; + push @rtext, sprintf " - %-4s (%2s) %-40s\n",$rsrc->{$key},$conn{$key},$connmsg{$key}; } } print @rtext; @@ -854,14 +819,15 @@ print " - SSL Client Binds: $sslClientBindCount\n"; print " - Failed SSL Client Binds: $sslClientFailedCount\n"; print " - SASL Binds: $saslBindCount\n"; if ($saslBindCount > 0){ - foreach $saslb ( sort {$saslmech{$b} <=> $saslmech{$a} } (keys %saslmech) ){ - printf " %-4s %-12s\n",$saslmech{$saslb}, $saslb; - } + my $saslmech = $hashes->{saslmech}; + foreach my $saslb ( sort {$saslmech->{$b} <=> $saslmech->{$a} } (keys %{$saslmech}) ){ + printf " %-4s %-12s\n",$saslmech->{$saslb}, $saslb; + } } print " - Directory Manager Binds: $rootDNBindCount\n"; print " - Anonymous Binds: $anonymousBindCount\n"; -$otherBindCount = $bindCount -($rootDNBindCount + $anonymousBindCount); +my $otherBindCount = $bindCount -($rootDNBindCount + $anonymousBindCount); print " - Other Binds: $otherBindCount\n\n"; ########################################################################## @@ -879,8 +845,10 @@ if ($verb eq "yes" || $usage =~ /y/){ print " (in seconds)\t\t<=1\t2\t3\t4-5\t6-10\t11-15\t>15\n"; print " --------------------------------------------------------------------------\n"; print " (# of connections)\t"; - for ($i=0; $i <=$#latency; $i++) { - print "$latency[$i]\t"; + for (my $i=0; $i <=$#latency; $i++) { + if (defined($latency[$i])) { + print "$latency[$i]\t"; + } } } @@ -891,9 +859,9 @@ if ($verb eq "yes" || $usage =~ /y/){ ################################### if ($verb eq "yes" || $usage =~ /p/){ - if ($openConnection[0] ne ""){ + if (@openConnection > 0){ print "\n\n----- Current Open Connection IDs ----- \n\n"; - for ($i=0; $i <= $#openConnection ; $i++) { + for (my $i=0; $i <= $#openConnection ; $i++) { if ($openConnection[$i]) { print "Conn Number: $i (" . getIPfromConn($i) . ")\n"; } @@ -910,17 +878,16 @@ if ($verb eq "yes" || $usage =~ /p/){ if ($usage =~ /e/i || $verb eq "yes"){ print "\n\n----- Errors -----\n"; - %er = sort( {$b <=> $a} %er); - for ($i = 0; $i<98; $i++){ - if ($err[$i] ne "" && $errorCode[$i] >0) { + for (my $i = 0; $i<98; $i++){ + if (defined($err[$i]) && $err[$i] ne "" && defined($errorCode[$i]) && $errorCode[$i] >0) { push @errtext, sprintf "%-8s %12s %-25s","err=$i",$errorCode[$i],$err[$i]; } } - for ($i = 0; $i < $#errtext; $i++){ - for ($ii = 0; $ii < $#errtext; $ii++){ - $yy="0"; - $zz="0"; + for (my $i = 0; $i < $#errtext; $i++){ + for (my $ii = 0; $ii < $#errtext; $ii++){ + my $yy="0"; + my $zz="0"; while ($errtext[$ii] =~ /(\w+)\s/g){ $errornum[$yy]="$1"; $yy++; @@ -936,7 +903,7 @@ if ($usage =~ /e/i || $verb eq "yes"){ } } } - for ($i = 0; $i <= $#errtext; $i++){ + for (my $i = 0; $i <= $#errtext; $i++){ $errtext[$i] =~ s/\n//g; print "\n" . $errtext[$i]; } @@ -953,51 +920,44 @@ if ($verb eq "yes" || $usage =~ /f/ ){ print "\n\n----- Top $sizeCount Failed Logins ------\n\n"; if ($ds6x eq "true"){ - %ds6xbadpwd = getCounterHashFromFile($DS6XBADPWD); - $ds6loop = 0; - foreach $ds6bp (sort { $ds6xbadpwd{$b} <=> $ds6xbadpwd{$a} } keys %ds6xbadpwd) { - if ($eloop > $sizeCount){ last; } - printf "%-4s %-40s\n", $ds6xbadpwd{$ds6bp}, $ds6bp; + my $ds6xbadpwd = $hashes->{ds6xbadpwd}; + my $ds6loop = 0; + foreach my $ds6bp (sort { $ds6xbadpwd->{$b} <=> $ds6xbadpwd->{$a} } keys %{$ds6xbadpwd}) { + if ($ds6loop > $sizeCount){ last; } + printf "%-4s %-40s\n", $ds6xbadpwd->{$ds6bp}, $ds6bp; $ds6loop++; } - undef %ds6xbadpwd; } else { - getInfoArraysFromFile($BINDINFO); - @bindVal = @fileArray1; - @bindConn = @fileArray2; - @bindOp = @fileArray3; - @badPasswordConn = getArrayFromFile($BADPWDCONN); - @badPasswordOp = getArrayFromFile($BADPWDOP); - @badPasswordIp = getArrayFromFile($BADPWDIP); - for ($ii =0 ; $ii < $badPwdCount; $ii++){ - for ($i = 0; $i < $bindCount; $i++){ - if ($badPasswordConn[$ii] eq $bindConn[$i] && $badPasswordOp[$ii] eq $bindOp[$i] ){ - $badPassword{ "$bindVal[$i]" } = $badPassword{ "$bindVal[$i]" } + 1; + my $bindVal = $arrays->{binddn}; + my $bindConn = $arrays->{bindconn}; + my $bindOp = $arrays->{bindop}; + my $badPasswordConn = $arrays->{badpwdconn}; + my $badPasswordOp = $arrays->{badpwdop}; + my $badPasswordIp = $arrays->{badpwdip}; + my %badPassword = (); + for (my $ii =0 ; $ii < $badPwdCount; $ii++){ + for (my $i = 0; $i < $bindCount; $i++){ + if ($badPasswordConn->[$ii] eq $bindConn->[$i] && $badPasswordOp->[$ii] eq $bindOp->[$i] ){ + $badPassword{ $bindVal->[$i] }++; } } } # sort the new hash of $badPassword{} - $bpTotal = 0; - $bpCount = 0; - foreach $badpw (sort {$badPassword{$b} <=> $badPassword{$a} } keys %badPassword){ + my $bpTotal = 0; + my $bpCount = 0; + foreach my $badpw (sort {$badPassword{$b} <=> $badPassword{$a} } keys %badPassword){ if ($bpCount > $sizeCount){ last;} $bpCount++; $bpTotal = $bpTotal + $badPassword{"$badpw"}; printf "%-4s %-40s\n", $badPassword{"$badpw"}, $badpw; } print "\nFrom the IP address(s) :\n\n"; - for ($i=0; $i<$badPwdCount; $i++) { - print "\t\t$badPasswordIp[$i]\n"; + for (my $i=0; $i<$badPwdCount; $i++) { + print "\t\t$badPasswordIp->[$i]\n"; } if ($bpTotal > $badPwdCount){ print "\n** Warning : Wrongly reported failed login attempts : ". ($bpTotal - $badPwdCount) . "\n"; } - undef @bindVal; - undef @bindConn; - undef @bindOp; - undef @badPasswordConn; - undef @badPasswordOp; - undef @badPasswordIp; } # this ends the if $ds6x = true } } @@ -1012,15 +972,14 @@ if ($verb eq "yes" || $usage =~ /f/ ){ if ($connCodeCount > 0){ if ($usage =~ /c/i || $verb eq "yes"){ print "\n\n----- Total Connection Codes -----\n\n"; - %conncount = &getCounterHashFromFile($CONNCOUNT); - - foreach $key (sort { $conncount{$b} <=> $conncount{$a} } keys %conncount) { - if ($conncount{$key} > 0){ - push @conntext, sprintf "%-4s %6s %-40s\n",$key,$conncount{$key},$connmsg{ $key }; + my $conncount = $hashes->{conncount}; + my @conntext; + foreach my $key (sort { $conncount->{$b} <=> $conncount->{$a} } keys %{$conncount}) { + if ($conncount->{$key} > 0){ + push @conntext, sprintf "%-4s %6s %-40s\n",$key,$conncount->{$key},$connmsg{ $key }; } } print @conntext; - undef %conncount; } } @@ -1031,35 +990,36 @@ if ($connCodeCount > 0){ ######################################## if ($usage =~ /i/i || $verb eq "yes"){ - %ip_hash = getTwoDimHashFromFile($IP_HASH); - %exCount = getCounterHashFromFile($EXCOUNT); - @ipkeys = keys %ip_hash; - @exxCount = keys %exCount; - $ip_count = ($#ipkeys + 1)-($#exxCount + 1); + my $ip_hash = $hashes->{ip_hash}; + my $exCount = $hashes->{excount}; + my @ipkeys = keys %{$ip_hash}; + my @exxCount = keys %${exCount}; + my $ip_count = ($#ipkeys + 1)-($#exxCount + 1); + my $ccount = 0; if ($ip_count > 0){ print "\n\n----- Top $sizeCount Clients -----\n\n"; print "Number of Clients: $ip_count\n\n"; - foreach $key (sort { $ip_hash{$b}{"count"} <=> $ip_hash{$a}{"count"} } keys %ip_hash) { - $exc = "no"; + foreach my $key (sort { $ip_hash->{$b} <=> $ip_hash->{$a} } @ipkeys) { + my $exc = "no"; if ($ccount > $sizeCount){ last;} $ccount++; - for ($xxx =0; $xxx <= $#excludeIP; $xxx++){ + for (my $xxx =0; $xxx < $#excludeIP; $xxx++){ if ($excludeIP[$xxx] eq $key){$exc = "yes";} } if ($exc ne "yes"){ - if ($ip_hash{ $key }{"count"} eq ""){$ip_hash{ $key }{"count"} = "?";} + if ($ip_hash->{ $key } eq ""){$ip_hash->{ $key } = "?";} printf "[%s] Client: %s\n",$ccount, $key; - printf "%10s - Connections\n", $ip_hash{ $key }{"count"}; - foreach $code (sort { $ip_hash{ $key }{$b} <=> $ip_hash{ $key }{$a} } keys %{$ip_hash{ $key }}) { + printf "%10s - Connections\n", $ip_hash->{ $key }; + my %counts; + map { $counts{$_} = $hashes->{$_}->{$key} if (defined($hashes->{$_}->{$key})) } @conncodes; + foreach my $code (sort { $counts{$b} <=> $counts{$a} } keys %counts) { if ($code eq 'count' ) { next; } - printf "%10s - %s (%s)\n", $ip_hash{ $key }{ $code }, $code, $connmsg{ $code }; + printf "%10s - %s (%s)\n", $counts{ $code }, $code, $connmsg{ $code }; } print "\n"; } } } - undef %exCount; - undef %ip_hash; } ################################### @@ -1069,21 +1029,22 @@ if ($usage =~ /i/i || $verb eq "yes"){ ################################### if ($usage =~ /b/i || $verb eq "yes"){ - %bindlist = getCounterHashFromFile($BINDLIST); - @bindkeys = keys %bindlist; - $bind_count = $#bindkeys + 1; + my $bindlist = $hashes->{bindlist}; + my @bindkeys = keys %{$bindlist}; + my $bind_count = $#bindkeys + 1; if ($bind_count > 0){ print "\n\n----- Top $sizeCount Bind DN's -----\n\n"; print "Number of Unique Bind DN's: $bind_count\n\n"; - $bindcount = 0; - foreach $dn (sort { $bindlist{$b} <=> $bindlist{$a} } keys %bindlist) { - if ($bindcount < $sizeCount){ - printf "%-8s %-40s\n", $bindlist{ $dn },$dn; - } + my $bindcount = 0; + foreach my $dn (sort { $bindlist->{$b} <=> $bindlist->{$a} } @bindkeys) { + if ($bindcount < $sizeCount){ + printf "%-8s %-40s\n", $bindlist->{ $dn },$dn; + } else { + last; + } $bindcount++; } } - undef %bindlist; } ######################################### @@ -1093,21 +1054,22 @@ if ($usage =~ /b/i || $verb eq "yes"){ ######################################### if ($usage =~ /a/i || $verb eq "yes"){ - %base = getCounterHashFromFile($BASE); - @basekeys = keys %base; - $base_count = $#basekeys + 1; + my $base = $hashes->{base}; + my @basekeys = keys %{$base}; + my $base_count = $#basekeys + 1; if ($base_count > 0){ print "\n\n----- Top $sizeCount Search Bases -----\n\n"; print "Number of Unique Search Bases: $base_count\n\n"; - $basecount = 0; - foreach $bas (sort { $base{$b} <=> $base{$a} } keys %base) { + my $basecount = 0; + foreach my $bas (sort { $base->{$b} <=> $base->{$a} } @basekeys) { if ($basecount < $sizeCount){ - printf "%-8s %-40s\n", $base{ $bas },$bas; - } + printf "%-8s %-40s\n", $base->{ $bas },$bas; + } else { + last; + } $basecount++; } } - undef %base; } ######################################### @@ -1117,21 +1079,22 @@ if ($usage =~ /a/i || $verb eq "yes"){ ######################################### if ($usage =~ /l/ || $verb eq "yes"){ - %filter = getCounterHashFromFile($FILTER); - @filterkeys = keys %filter; - $filter_count = $#filterkeys + 1; + my $filter = $hashes->{filter}; + my @filterkeys = keys %{$filter}; + my $filter_count = $#filterkeys + 1; if ($filter_count > 0){ print "\n\n----- Top $sizeCount Search Filters -----\n"; print "\nNumber of Unique Search Filters: $filter_count\n\n"; - $filtercount = 0; - foreach $filt (sort { $filter{$b} <=> $filter{$a} } keys %filter){ + my $filtercount = 0; + foreach my $filt (sort { $filter->{$b} <=> $filter->{$a} } @filterkeys){ if ($filtercount < $sizeCount){ - printf "%-8s %-40s\n", $filter{$filt}, $filt; - } + printf "%-8s %-40s\n", $filter->{$filt}, $filt; + } else { + last; + } $filtercount++; } } - undef %filter; } ######################################### @@ -1140,20 +1103,23 @@ if ($usage =~ /l/ || $verb eq "yes"){ # # ######################################### +my $first; if ($usage =~ /t/i || $verb eq "yes"){ - %etime = getCounterHashFromFile($ETIME); + my $etime = $hashes->{etime}; + my @ekeys = keys %{$etime}; # # print most often etimes # print "\n\n----- Top $sizeCount Most Frequent etimes -----\n\n"; - $eloop = 0; - foreach $et (sort { $etime{$b} <=> $etime{$a} } keys %etime) { + my $eloop = 0; + my $retime = 0; + foreach my $et (sort { $etime->{$b} <=> $etime->{$a} } @ekeys) { if ($eloop == $sizeCount) { last; } if ($retime ne "2"){ $first = $et; $retime = "2"; } - printf "%-8s %-12s\n", $etime{ $et }, "etime=$et"; + printf "%-8s %-12s\n", $etime->{ $et }, "etime=$et"; $eloop++; } # @@ -1161,12 +1127,11 @@ if ($usage =~ /t/i || $verb eq "yes"){ # print "\n\n----- Top $sizeCount Longest etimes -----\n\n"; $eloop = 0; - foreach $et (sort { $b <=> $a } (keys %etime)) { + foreach my $et (sort { $b <=> $a } @ekeys) { if ($eloop == $sizeCount) { last; } - printf "%-12s %-10s\n","etime=$et",$etime{ $et }; + printf "%-12s %-10s\n","etime=$et",$etime->{ $et }; $eloop++; } - undef %etime; } ####################################### @@ -1177,23 +1142,23 @@ if ($usage =~ /t/i || $verb eq "yes"){ if ($usage =~ /n/i || $verb eq "yes"){ - %nentries = getCounterHashFromFile($NENTRIES); + my $nentries = $hashes->{nentries}; + my @nkeys = keys %{$nentries}; print "\n\n----- Top $sizeCount Largest nentries -----\n\n"; - $eloop = 0; - foreach $nentry (sort { $b <=> $a } (keys %nentries)){ + my $eloop = 0; + foreach my $nentry (sort { $b <=> $a } @nkeys){ if ($eloop == $sizeCount) { last; } - printf "%-18s %12s\n","nentries=$nentry", $nentries{ $nentry }; + printf "%-18s %12s\n","nentries=$nentry", $nentries->{ $nentry }; $eloop++; } print "\n\n----- Top $sizeCount Most returned nentries -----\n\n"; $eloop = 0; - foreach $nentry (sort { $nentries{$b} <=> $nentries{$a} } (keys %nentries)){ + foreach my $nentry (sort { $nentries->{$b} <=> $nentries->{$a} } @nkeys){ if ($eloop == $sizeCount) { last; } - printf "%-12s %-14s\n", $nentries{ $nentry }, "nentries=$nentry"; + printf "%-12s %-14s\n", $nentries->{ $nentry }, "nentries=$nentry"; $eloop++; } print "\n"; - undef %nentries; } ########################################## @@ -1204,9 +1169,10 @@ if ($usage =~ /n/i || $verb eq "yes"){ if ($usage =~ /x/i || $verb eq "yes"){ if ($extopCount > 0){ - %oid = getCounterHashFromFile($OID); + my $oid = $hashes->{oid}; print "\n\n----- Extended Operations -----\n\n"; - foreach $oids (sort { $oid{$b} <=> $oid{$a} } (keys %oid) ){ + foreach my $oids (sort { $oid->{$b} <=> $oid->{$a} } (keys %{$oid}) ){ + my $oidmessage; if ($oids eq "2.16.840.1.113730.3.5.1"){ $oidmessage = "Transaction Request"} #depreciated? elsif ($oids eq "2.16.840.1.113730.3.5.2"){ $oidmessage = "Transaction Response"} #depreciated? elsif ($oids eq "2.16.840.1.113730.3.5.3"){ $oidmessage = "Start Replication Request (incremental update)"} @@ -1236,9 +1202,8 @@ if ($usage =~ /x/i || $verb eq "yes"){ elsif ($oids eq "1.3.6.1.4.1.4203.1.11.1"){ $oidmessage = "Password Modify"} elsif ($oids eq "2.16.840.1.113730.3.4.20"){ $oidmessage = "MTN Control Use One Backend"} else {$oidmessage = "Other"} - printf "%-6s %-23s %-60s\n", $oid{ $oids }, $oids, $oidmessage; + printf "%-6s %-23s %-60s\n", $oid->{ $oids }, $oids, $oidmessage; } - undef %oid; } } @@ -1250,15 +1215,14 @@ if ($usage =~ /x/i || $verb eq "yes"){ if ($usage =~ /r/i || $verb eq "yes"){ if ($anyAttrs > 0){ - %attr = getCounterHashFromFile($ATTR); + my $attr = $hashes->{attr}; print "\n\n----- Top $sizeCount Most Requested Attributes -----\n\n"; - $eloop = 0; - foreach $mostAttr (sort { $attr{$b} <=> $attr{$a} } (keys %attr) ){ + my $eloop = 0; + foreach my $mostAttr (sort { $attr->{$b} <=> $attr->{$a} } (keys %{$attr}) ){ if ($eloop eq $sizeCount){ last; } - printf "%-10s %-19s\n", $attr{$mostAttr}, $mostAttr; + printf "%-10s %-19s\n", $attr->{$mostAttr}, $mostAttr; $eloop++; } - undef %attr; } } @@ -1269,77 +1233,59 @@ if ($usage =~ /r/i || $verb eq "yes"){ ############################# if ($usage =~ /g/i || $verb eq "yes"){ - $abandonTotal = $srchCount + $delCount + $modCount + $addCount + $modrdnCount + $bindCount + $extopCount + $cmpCount; + my $abandonTotal = $srchCount + $delCount + $modCount + $addCount + $modrdnCount + $bindCount + $extopCount + $cmpCount; if ($verb eq "yes" && $abandonCount > 0 && $abandonTotal > 0){ - %conn_hash = getHashFromFile($CONN_HASH); - @srchConn = getArrayFromFile($SRCH_CONN); - @srchOp = getArrayFromFile($SRCH_OP); - @delConn = getArrayFromFile($DEL_CONN); - @delOp = getArrayFromFile($DEL_OP); - @targetConn = getArrayFromFile($TARGET_CONN); - @targetOp = getArrayFromFile($TARGET_OP); - @msgid = getArrayFromFile($MSGID); - @addConn = getArrayFromFile($ADD_CONN); - @addOp = getArrayFromFile($ADD_OP); - @modConn = getArrayFromFile($MOD_CONN); - @modOp = getArrayFromFile($MOD_OP); - @cmpConn = getArrayFromFile($CMP_CONN); - @cmpOp = getArrayFromFile($CMP_OP); - @modrdnConn = getArrayFromFile($MODRDN_CONN); - @modrdnOp = getArrayFromFile($MODRDN_OP); - @bindConn = getArrayFromFile($BIND_CONN); - @bindOp = getArrayFromFile($BIND_OP); - @unbindConn = getArrayFromFile($UNBIND_CONN); - @unbindOp = getArrayFromFile($UNBIND_OP); - @extConn = getArrayFromFile($EXT_CONN); - @extOp = getArrayFromFile($EXT_OP); + my $conn_hash = $hashes->{conn_hash}; print "\n\n----- Abandon Request Stats -----\n\n"; - for ($g = 0; $g < $abandonCount; $g++){ - for ($sc = 0; $sc < $srchCount; $sc++){ - if ($srchConn[$sc] eq $targetConn[$g] && $srchOp[$sc] eq $targetOp[$g] ){ - print " - SRCH conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; + for (my $g = 0; $g < $abandonCount; $g++){ + my $conn = $arrays->{targetconn}->[$g]; + my $op = $arrays->{targetop}->[$g]; + my $msgid = $arrays->{msgid}->[$g]; + for (my $sc = 0; $sc < $srchCount; $sc++){ + if ($arrays->{srchconn}->[$sc] eq $conn && $arrays->{srchop}->[$sc] eq $op ){ + print " - SRCH conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; } } - for ($dc = 0; $dc < $delCount; $dc++){ - if ($delConn[$dc] eq $targetConn[$g] && $delOp[$dc] eq $targetOp[$g]){ - print " - DEL conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; + for (my $dc = 0; $dc < $delCount; $dc++){ + if ($arrays->{delconn}->[$dc] eq $conn && $arrays->{delop}->[$dc] eq $op){ + print " - DEL conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; } } - for ($adc = 0; $adc < $addCount; $adc++){ - if ($addConn[$adc] eq $targetConn[$g] && $addOp[$adc] eq $targetOp[$g]){ - print " - ADD conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; + for (my $adc = 0; $adc < $addCount; $adc++){ + if ($arrays->{addconn}->[$adc] eq $conn && $arrays->{addop}->[$adc] eq $op){ + print " - ADD conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; } } - for ($mc = 0; $mc < $modCount; $mc++){ - if ($modConn[$mc] eq $targetConn[$g] && $modOp[$mc] eq $targetOp[$g]){ - print " - MOD conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; + for (my $mc = 0; $mc < $modCount; $mc++){ + if ($arrays->{modconn}->[$mc] eq $conn && $arrays->{modop}->[$mc] eq $op){ + print " - MOD conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; } } - for ($cc = 0; $cc < $cmpCount; $cc++){ - if ($cmpConn[$mdc] eq $targetConn[$g] && $cmpOp[$mdc] eq $targetOp[$g]){ - print " - CMP conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; + for (my $cc = 0; $cc < $cmpCount; $cc++){ + if ($arrays->{cmpconn}->[$cc] eq $conn && $arrays->{cmpop}->[$cc] eq $op){ + print " - CMP conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; } } - for ($mdc = 0; $mdc < $modrdnCount; $mdc++){ - if ($modrdnConn[$mdc] eq $targetConn[$g] && $modrdnOp[$mdc] eq $targetOp[$g]){ - print " - MODRDN conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; + for (my $mdc = 0; $mdc < $modrdnCount; $mdc++){ + if ($arrays->{modrdnconn}->[$mdc] eq $conn && $arrays->{modrdnop}->[$mdc] eq $op){ + print " - MODRDN conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; } } - for ($bcb = 0; $bcb < $bindCount; $bcb++){ - if ($bindConn[$bcb] eq $targetConn[$g] && $bindOp[$bcb] eq $targetOp[$g]){ - print " - BIND conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; + for (my $bcb = 0; $bcb < $bindCount; $bcb++){ + if ($arrays->{bindconn}->[$bcb] eq $conn && $arrays->{bindop}->[$bcb] eq $op){ + print " - BIND conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; } } - for ($ubc = 0; $ubc < $unbindCount; $ubc++){ - if ($unbindConn[$ubc] eq $targetConn[$g] && $unbindOp[$ubc] eq $targetOp[$g]){ - print " - UNBIND conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; + for (my $ubc = 0; $ubc < $unbindCount; $ubc++){ + if ($arrays->{unbindconn}->[$ubc] eq $conn && $arrays->{unbindop}->[$ubc] eq $op){ + print " - UNBIND conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; } } - for ($ec = 0; $ec < $extopCount; $ec++){ - if ($extConn[$ec] eq $targetConn[$g] && $extOp[$ec] eq $targetOp[$g]){ - print " - EXT conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; + for (my $ec = 0; $ec < $extopCount; $ec++){ + if ($arrays->{extconn}->[$ec] eq $conn && $arrays->{extop}->[$ec] eq $op){ + print " - EXT conn=$conn op=$op msgid=$msgid client=$conn_hash->{$conn}\n"; } } } @@ -1354,9 +1300,9 @@ print "\n"; ####################################### if ($usage =~ /j/i || $verb eq "yes"){ - %conncount = getCounterHashFromFile($CONNCOUNT); + my $conncount = $hashes->{conncount}; print "\n----- Recommendations -----\n"; - $recCount = "1"; + my $recCount = "1"; if ($unindexedSrchCountNotesA > 0){ print "\n $recCount. You have unindexed searches, this can be caused from a search on an unindexed attribute, or your returned results exceeded the allidsthreshold. Unindexed searches are not recommended. To refuse unindexed searches, switch \'nsslapd-require-index\' to \'on\' under your database entry (e.g. cn=UserRoot,cn=ldbm database,cn=plugins,cn=config).\n"; $recCount++; @@ -1365,11 +1311,11 @@ if ($usage =~ /j/i || $verb eq "yes"){ print "\n $recCount. You have unindexed components, this can be caused from a search on an unindexed attribute, or your returned results exceeded the allidsthreshold. Unindexed components are not recommended. To refuse unindexed searches, switch \'nsslapd-require-index\' to \'on\' under your database entry (e.g. cn=UserRoot,cn=ldbm database,cn=plugins,cn=config).\n"; $recCount++; } - if ($conncount{"T1"} > 0){ + if (defined($conncount->{"T1"}) and $conncount->{"T1"} > 0){ print "\n $recCount. You have some connections that are are being closed by the idletimeout setting. You may want to increase the idletimeout if it is set low.\n"; $recCount++; } - if ($conncount{"T2"} > 0){ + if (defined($conncount->{"T2"}) and $conncount->{"T2"} > 0){ print "\n $recCount. You have some coonections that are being closed by the ioblocktimeout setting. You may want to increase the ioblocktimeout.\n"; $recCount++; } @@ -1391,7 +1337,7 @@ if ($usage =~ /j/i || $verb eq "yes"){ print "\n $recCount. You have more unsuccessful operations than successful operations. You should investigate this difference.\n"; $recCount++; } - if ($conncount{"U1"} < ($connCodeCount - $conncount{"U1"})){ + if (defined($conncount->{"U1"}) and $conncount->{"U1"} < ($connCodeCount - $conncount->{"U1"})){ print "\n $recCount. You have more abnormal connection codes than cleanly closed connections. You may want to investigate this difference.\n"; $recCount++; } @@ -1407,7 +1353,6 @@ if ($usage =~ /j/i || $verb eq "yes"){ print "\nNone.\n"; } print "\n"; - undef %conncount; } # @@ -1499,11 +1444,9 @@ parseLineBind { $linesProcessed++; $lineBlockCount++; local $_ = $logline; + my $ip; - if ($lineBlockCount >= $limit){ - print STDERR sprintf" %10s Lines Processed\n",$linesProcessed; - $lineBlockCount="0"; - } + statusreport(); # skip blank lines return if $_ =~ /^\s/; @@ -1531,7 +1474,8 @@ parseLineBind { } } if ($_ =~ /connection from *([0-9A-Fa-f\.\:]+)/i ) { - for ($excl =0; $excl <= $#excludeIP; $excl++){ + my $skip = "yes"; + for (my $excl =0; $excl < $#excludeIP; $excl++){ if ($excludeIP[$excl] eq $1){ $skip = "yes"; last; @@ -1547,6 +1491,7 @@ parseLineBind { return; } if (/ BIND/ && $_ =~ /dn=\"(.*)\" method/i ){ + my $dn; if ($1 eq ""){ $dn = "Anonymous"; } else { @@ -1599,30 +1544,36 @@ parseLineBind { sub processOpForBindReport { - $op = @_[0]; - $data = @_[1]; + my $op = shift; + my $data = shift; if ($data =~ /conn= *([0-9]+)/i) { - foreach $dn (keys %bindReport){ + foreach my $dn (keys %bindReport){ if ($bindReport{$dn}{"conn"} =~ / $1 /){ - $bindDN = $dn; - $bindReport{$bindDN}{$op}++; + $bindReport{$dn}{$op}++; return; } } } } +my ($last_tm, $lastzone, $last_min, $gmtime, $tzoff); sub parseLineNormal { local $_ = $logline; + my $ip; + my $tmpp; + my $exc = "no"; + my $connID; + my $con; + my $op; + $linesProcessed++; + $lineBlockCount++; # lines starting blank are restart return if $_ =~ /^\s/; - $linesProcessed++; - $lineBlockCount++; - if ($lineBlockCount >= $limit){ print STDERR sprintf" %10s Lines Processed\n",$linesProcessed; $lineBlockCount="0";} + statusreport(); # gather/process the timestamp if($firstFile == 1 && $_ =~ /^\[/){ @@ -1648,30 +1599,30 @@ sub parseLineNormal } # Additional performance stats - ($time, $tzone) = split (' ', $_); - if ($reportStats && $time ne $last_tm) + my ($time, $tzone) = split (' ', $_); + if (($reportStats or ($verb eq "yes") || ($usage =~ /y/)) && (!defined($last_tm) or ($time ne $last_tm))) { $last_tm = $time; $time =~ s/\[//; $tzone =~ s/\].*//; - if($tzone ne $lastzone) + if(!defined($lastzone) or $tzone ne $lastzone) { # tz offset change $lastzone=$tzone; - ($sign,$hr,$min) = $tzone =~ m/(.)(\d\d)(\d\d)/; + my ($sign,$hr,$min) = $tzone =~ m/(.)(\d\d)(\d\d)/; $tzoff = $hr*3600 + $min*60; $tzoff *= -1 if $sign eq '-'; # to be subtracted from converted values. } - ($date, $hr, $min, $sec) = split (':', $time); - ($day, $mon, $yr) = split ('/', $date); - $newmin = timegm(0, $min, $hr, $day, $monthname{$mon}, $yr) - $tzoff; + my ($date, $hr, $min, $sec) = split (':', $time); + my ($day, $mon, $yr) = split ('/', $date); + my $newmin = timegm(0, $min, $hr, $day, $monthname{$mon}, $yr) - $tzoff; $gmtime = $newmin + $sec; print_stats_block( $s_stats ); reset_stats_block( $s_stats, $gmtime, $time.' '.$tzone ); - if ($newmin != $last_min) + if (!defined($last_min) or $newmin != $last_min) { print_stats_block( $m_stats ); $time =~ s/\d\d$/00/; @@ -1689,58 +1640,57 @@ sub parseLineNormal if($reportStats){ inc_stats('srch',$s_stats,$m_stats); } if ($_ =~ / attrs=\"(.*)\"/i){ $anyAttrs++; - $attrs = $1 . " "; - while ($attrs =~ /(\S+)\s/g){ - writeFile($ATTR, $1); - } - } + my $attr = $hashes->{attr}; + map { $attr->{$_}++ } split /\s/, $1; + } if (/ attrs=ALL/){ - writeFile($ATTR, "All Attributes"); + my $attr = $hashes->{attr}; + $attr->{"All Attributes"}++; $anyAttrs++; } - if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ writeFile($SRCH_CONN, $1);} - if ($_ =~ /op= *([0-9]+)/i){ writeFile($SRCH_OP, $1);} + if ($verb eq "yes"){ + if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{srchconn}}, $1;} + if ($_ =~ /op= *([0-9]+)/i){ push @{$arrays->{srchop}}, $1;} } } if (m/ DEL/){ $delCount++; if($reportStats){ inc_stats('del',$s_stats,$m_stats); } if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ writeFile($DEL_CONN, $1);} - if ($_ =~ /op= *([0-9]+)/i){ writeFile($DEL_OP, $1);} + if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{delconn}}, $1;} + if ($_ =~ /op= *([0-9]+)/i){ push @{$arrays->{delop}}, $1;} } } if (m/ MOD dn=/){ $modCount++; if($reportStats){ inc_stats('mod',$s_stats,$m_stats); } if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ writeFile($MOD_CONN, $1);} - if ($_ =~ /op= *([0-9]+)/i){ writeFile($MOD_OP, $1); } + if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{modconn}}, $1;} + if ($_ =~ /op= *([0-9]+)/i){ push @{$arrays->{modop}}, $1; } } } if (m/ ADD/){ $addCount++; if($reportStats){ inc_stats('add',$s_stats,$m_stats); } if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ writeFile($ADD_CONN, $1); } - if ($_ =~ /op= *([0-9]+)/i){ writeFile($ADD_OP, $1); } + if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{addconn}}, $1; } + if ($_ =~ /op= *([0-9]+)/i){ push @{$arrays->{addop}}, $1; } } } if (m/ MODRDN/){ $modrdnCount++; if($reportStats){ inc_stats('modrdn',$s_stats,$m_stats); } if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ writeFile($MODRDN_CONN, $1); } - if ($_ =~ /op= *([0-9]+)/i){ writeFile($MODRDN_OP, $1); } + if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{modrdnconn}}, $1; } + if ($_ =~ /op= *([0-9]+)/i){ push @{$arrays->{modrdnop}}, $1; } } } if (m/ CMP dn=/){ $cmpCount++; if($reportStats){ inc_stats('cmp',$s_stats,$m_stats); } if ($verb eq "yes" || $usage =~ /g/i){ - if ($_ =~ /conn= *([0-9]+)/i){ writeFile($CMP_CONN, $1);} - if ($_ =~ /op= *([0-9]+)/i){ writeFile($CMP_OP, $1);} + if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{cmpconn}}, $1;} + if ($_ =~ /op= *([0-9]+)/i){ push @{$arrays->{cmpop}}, $1;} } } if (m/ ABANDON /){ @@ -1748,9 +1698,9 @@ sub parseLineNormal if($reportStats){ inc_stats('abandon',$s_stats,$m_stats); } $allResults++; if ($_ =~ /targetop= *([0-9a-zA-Z]+)/i ){ - writeFile($TARGET_OP, $1); - if ($_ =~ /conn= *([0-9]+)/i){ writeFile($TARGET_CONN, $1); } - if ($_ =~ /msgid= *([0-9]+)/i){ writeFile($MSGID, $1);} + push @{$arrays->{targetop}}, $1; + if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{targetconn}}, $1; } + if ($_ =~ /msgid= *([0-9]+)/i){ push @{$arrays->{msgid}}, $1; } } } if (m/ VLV /){ @@ -1774,20 +1724,19 @@ sub parseLineNormal if ($1 ne ""){ $tmpp = $1; $tmpp =~ tr/A-Z/a-z/; - writeFile($BINDLIST, $tmpp); + $hashes->{bindlist}->{$tmpp}++; if($1 eq $rootDN){ $rootDNBindCount++; } } else { $anonymousBindCount++; - writeFile($BINDLIST, "Anonymous Binds"); + $hashes->{bindlist}->{"Anonymous Binds"}++; inc_stats('anonbind',$s_stats,$m_stats); } } if (m/ connection from/){ - $exc = "no"; if ($_ =~ /connection from *([0-9A-Fa-f\.\:]+)/i ){ - for ($xxx =0; $xxx <= $#excludeIP; $xxx++){ + for (my $xxx =0; $xxx < $#excludeIP; $xxx++){ if ($excludeIP[$xxx] eq $1){$exc = "yes";} } if ($exc ne "yes"){ @@ -1801,11 +1750,13 @@ sub parseLineNormal } ($connID) = $_ =~ /conn=(\d*)\s/; $openConnection[$connID]++; - ($time, $tzone) = split (' ', $_); - ($date, $hr, $min, $sec) = split (':', $time); - ($day, $mon, $yr) = split ('/', $date); - $day =~ s/\[//; - $start_time_of_connection[$connID] = timegm($sec, $min, $hours, $day, $monthname{$mon}, $yr); + if ($reportStats or ($verb eq "yes") || ($usage =~ /y/)) { + my ($time, $tzone) = split (' ', $_); + my ($date, $hr, $min, $sec) = split (':', $time); + my ($day, $mon, $yr) = split ('/', $date); + $day =~ s/\[//; + $start_time_of_connection[$connID] = timegm($sec, $min, $hr, $day, $monthname{$mon}, $yr); + } } if (m/ SSL client bound as /){$sslClientBindCount++;} if (m/ SSL failed to map client certificate to LDAP DN/){$sslClientFailedCount++;} @@ -1816,16 +1767,22 @@ sub parseLineNormal ($connID) = $_ =~ /conn=(\d*)\s/; $openConnection[$connID]--; - $end_time_of_connection[$connID] = $gmtime; - $diff = $end_time_of_connection[$connID] - $start_time_of_connection[$connID]; - $start_time_of_connection[$connID] = $end_time_of_connection[$connID] = 0; - if ($diff <= 1) { $latency[0]++;} - if ($diff == 2) { $latency[1]++;} - if ($diff == 3) { $latency[2]++;} - if ($diff >= 4 && $diff <=5 ) { $latency[3]++;} - if ($diff >= 6 && $diff <=10 ) { $latency[4]++;} - if ($diff >= 11 && $diff <=15 ) { $latency[5]++;} - if ($diff >= 16) { $latency[6] ++;} + if ($reportStats or ($verb eq "yes") || ($usage =~ /y/)) { + # if we didn't see the start time of this connection + # i.e. due to truncation or log rotation + # then just set to 0 + my $stoc = $start_time_of_connection[$connID] || 0; + $end_time_of_connection[$connID] = $gmtime || 0; + my $diff = $end_time_of_connection[$connID] - $stoc; + $start_time_of_connection[$connID] = $end_time_of_connection[$connID] = 0; + if ($diff <= 1) { $latency[0]++;} + if ($diff == 2) { $latency[1]++;} + if ($diff == 3) { $latency[2]++;} + if ($diff >= 4 && $diff <=5 ) { $latency[3]++;} + if ($diff >= 6 && $diff <=10 ) { $latency[4]++;} + if ($diff >= 11 && $diff <=15 ) { $latency[5]++;} + if ($diff >= 16) { $latency[6] ++;} + } } if (m/ BIND/ && $_ =~ /dn=\"(.*)\" method/i ){ if($reportStats){ inc_stats('bind',$s_stats,$m_stats); } @@ -1834,25 +1791,26 @@ sub parseLineNormal if($1 eq $rootDN){$rootDNBindCount++;} $tmpp = $1; $tmpp =~ tr/A-Z/a-z/; - writeFile($BINDLIST, $tmpp); - $bindVal = $tmpp; - if ($_ =~ /conn= *([0-9]+)/i) { $bindConn = $1; writeFile($BIND_CONN, $1);} - if ($_ =~ /op= *([0-9]+)/i) { $bindOp = $1; writeFile($BIND_OP, $1);} + $hashes->{bindlist}->{$tmpp}++; + if ($_ =~ /conn= *([0-9]+)/i) { push @{$arrays->{bindconn}}, $1;} + if ($_ =~ /op= *([0-9]+)/i) { push @{$arrays->{bindop}}, $1;} if($usage =~ /f/ || $verb eq "yes"){ - # only need this for the failed bind report - writeFile($BINDINFO, "$bindVal ,, $bindConn ,, $bindOp"); + push @{$arrays->{binddn}}, $tmpp; } } else { $anonymousBindCount++; - writeFile($BINDLIST, "Anonymous Binds"); + $hashes->{bindlist}->{"Anonymous Binds"}++; + if ($_ =~ /conn= *([0-9]+)/i) { push @{$arrays->{bindconn}}, $1;} + if ($_ =~ /op= *([0-9]+)/i) { push @{$arrays->{bindop}}, $1;} + push @{$arrays->{binddn}}, ""; inc_stats('anonbind',$s_stats,$m_stats); } } if (m/ UNBIND/){ $unbindCount++; if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ writeFile($UNBIND_CONN, $1); } - if ($_ =~ /op= *([0-9]+)/i){ writeFile($UNBIND_OP, $1); } + if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{unbindconn}}, $1; } + if ($_ =~ /op= *([0-9]+)/i){ push @{$arrays->{unbindop}}, $1; } } } if (m/ RESULT err=/ && m/ notes=P/){ @@ -1863,7 +1821,7 @@ sub parseLineNormal $con = $1; if ($_ =~ /op= *([0-9]+)/i){ $op = $1;} } - for ($i=0; $i <= $vlvCount;$i++){ + for (my $i=0; $i <= $vlvCount;$i++){ if ($vlvconn[$i] eq $con && $vlvop[$i] eq $op){ $vlvNotesACount++; $isVlvNotes="1";} } if($isVlvNotes == 0){ @@ -1873,22 +1831,12 @@ sub parseLineNormal if($reportStats){ inc_stats('notesA',$s_stats,$m_stats); } } if ($usage =~ /u/ || $verb eq "yes"){ - if ($isVlvNnotes == 0 ){ - if ($_ =~ /etime= *([0-9.]+)/i ){ - writeFile($NOTES_A_ETIME, $1); - } - if ($_ =~ /conn= *([0-9]+)/i){ - writeFile($NOTES_A_CONN, $1); - } - if ($_ =~ /op= *([0-9]+)/i){ - writeFile($NOTES_A_OP, $1); - } - if ($_ =~ / *([0-9a-z:\/]+)/i){ - writeFile($NOTES_A_TIME, $1); - } - if ($_ =~ /nentries= *([0-9]+)/i ){ - writeFile($NOTES_A_NENTRIES, $1); - } + if ($isVlvNotes == 0 ){ + if ($_ =~ /etime= *([0-9.]+)/i ){ push @{$arrays->{notesAetime}}, $1; } + if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{notesAconn}}, $1; } + if ($_ =~ /op= *([0-9]+)/i){ push @{$arrays->{notesAop}}, $1; } + if ($_ =~ / *([0-9a-z:\/]+)/i){ push @{$arrays->{notesAtime}}, $1; } + if ($_ =~ /nentries= *([0-9]+)/i ){ push @{$arrays->{notesAnentries}}, $1; } } } $isVlvNotes = 0; @@ -1898,7 +1846,7 @@ sub parseLineNormal $con = $1; if ($_ =~ /op= *([0-9]+)/i){ $op = $1;} } - for ($i=0; $i <= $vlvCount;$i++){ + for (my $i=0; $i <= $vlvCount;$i++){ if ($vlvconn[$i] eq $con && $vlvop[$i] eq $op){ $vlvNotesUCount++; $isVlvNotes="1";} } if($isVlvNotes == 0){ @@ -1908,85 +1856,75 @@ sub parseLineNormal if($reportStats){ inc_stats('notesU',$s_stats,$m_stats); } } if ($usage =~ /u/ || $verb eq "yes"){ - if ($isVlvNnotes == 0 ){ - if ($_ =~ /etime= *([0-9.]+)/i ){ - writeFile($NOTES_U_ETIME, $1); - } - if ($_ =~ /conn= *([0-9]+)/i){ - writeFile($NOTES_U_CONN, $1); - } - if ($_ =~ /op= *([0-9]+)/i){ - writeFile($NOTES_U_OP, $1); - } - if ($_ =~ / *([0-9a-z:\/]+)/i){ - writeFile($NOTES_U_TIME, $1); - } - if ($_ =~ /nentries= *([0-9]+)/i ){ - writeFile($NOTES_U_NENTRIES, $1); - } + if ($isVlvNotes == 0 ){ + if ($_ =~ /etime= *([0-9.]+)/i ){ push @{$arrays->{notesUetime}}, $1; } + if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{notesUconn}}, $1; } + if ($_ =~ /op= *([0-9]+)/i){ push @{$arrays->{notesUop}}, $1; } + if ($_ =~ / *([0-9a-z:\/]+)/i){ push @{$arrays->{notesUtime}}, $1; } + if ($_ =~ /nentries= *([0-9]+)/i ){ push @{$arrays->{notesUnentries}}, $1; } } } $isVlvNotes = 0; } if (m/ closed error 32/){ $brokenPipeCount++; - if (m/- T1/){ writeFile($RC,"T1"); } - elsif (m/- T2/){ writeFile($RC,"T2"); } - elsif (m/- A1/){ writeFile($RC,"A1"); } - elsif (m/- B1/){ writeFile($RC,"B1"); } - elsif (m/- B4/){ writeFile($RC,"B4"); } - elsif (m/- B2/){ writeFile($RC,"B2"); } - elsif (m/- B3/){ writeFile($RC,"B3"); } - elsif (m/- R1/){ writeFile($RC,"R1"); } - elsif (m/- P1/){ writeFile($RC,"P1"); } - elsif (m/- P1/){ writeFile($RC,"P2"); } - elsif (m/- U1/){ writeFile($RC,"U1"); } - else { writeFile($RC,"other"); } + if (m/- T1/){ $hashes->{rc}->{"T1"}++; } + elsif (m/- T2/){ $hashes->{rc}->{"T2"}++; } + elsif (m/- A1/){ $hashes->{rc}->{"A1"}++; } + elsif (m/- B1/){ $hashes->{rc}->{"B1"}++; } + elsif (m/- B4/){ $hashes->{rc}->{"B4"}++; } + elsif (m/- B2/){ $hashes->{rc}->{"B2"}++; } + elsif (m/- B3/){ $hashes->{rc}->{"B3"}++; } + elsif (m/- R1/){ $hashes->{rc}->{"R1"}++; } + elsif (m/- P1/){ $hashes->{rc}->{"P1"}++; } + elsif (m/- P1/){ $hashes->{rc}->{"P2"}++; } + elsif (m/- U1/){ $hashes->{rc}->{"U1"}++; } + else { $hashes->{rc}->{"other"}++; } } if (m/ closed error 131/ || m/ closed error -5961/){ $connResetByPeerCount++; - if (m/- T1/){ writeFile($SRC,"T1"); } - elsif (m/- T2/){ writeFile($SRC,"T2"); } - elsif (m/- A1/){ writeFile($SRC,"A1"); } - elsif (m/- B1/){ writeFile($SRC,"B1"); } - elsif (m/- B4/){ writeFile($SRC,"B4"); } - elsif (m/- B2/){ writeFile($SRC,"B2"); } - elsif (m/- B3/){ writeFile($SRC,"B3"); } - elsif (m/- R1/){ writeFile($SRC,"R1"); } - elsif (m/- P1/){ writeFile($SRC,"P1"); } - elsif (m/- P1/){ writeFile($SRC,"P2"); } - elsif (m/- U1/){ writeFile($SRC,"U1"); } - else { writeFile($SRC,"other"); } + if (m/- T1/){ $hashes->{src}->{"T1"}++; } + elsif (m/- T2/){ $hashes->{src}->{"T2"}++; } + elsif (m/- A1/){ $hashes->{src}->{"A1"}++; } + elsif (m/- B1/){ $hashes->{src}->{"B1"}++; } + elsif (m/- B4/){ $hashes->{src}->{"B4"}++; } + elsif (m/- B2/){ $hashes->{src}->{"B2"}++; } + elsif (m/- B3/){ $hashes->{src}->{"B3"}++; } + elsif (m/- R1/){ $hashes->{src}->{"R1"}++; } + elsif (m/- P1/){ $hashes->{src}->{"P1"}++; } + elsif (m/- P1/){ $hashes->{src}->{"P2"}++; } + elsif (m/- U1/){ $hashes->{src}->{"U1"}++; } + else { $hashes->{src}->{"other"}++; } } if (m/ closed error 11/){ $resourceUnavailCount++; - if (m/- T1/){ writeFile($RSRC,"T1"); } - elsif (m/- T2/){ writeFile($RSRC,"T2"); } - elsif (m/- A1/){ writeFile($RSRC,"A1"); } - elsif (m/- B1/){ writeFile($RSRC,"B1"); } - elsif (m/- B4/){ writeFile($RSRC,"B4"); } - elsif (m/- B2/){ writeFile($RSRC,"B2"); } - elsif (m/- B3/){ writeFile($RSRC,"B3"); } - elsif (m/- R1/){ writeFile($RSRC,"R1"); } - elsif (m/- P1/){ writeFile($RSRC,"P1"); } - elsif (m/- P1/){ writeFile($RSRC,"P2"); } - elsif (m/- U1/){ writeFile($RSRC,"U1"); } - else { writeFile($RSRC,"other"); } + if (m/- T1/){ $hashes->{rsrc}->{"T1"}++; } + elsif (m/- T2/){ $hashes->{rsrc}->{"T2"}++; } + elsif (m/- A1/){ $hashes->{rsrc}->{"A1"}++; } + elsif (m/- B1/){ $hashes->{rsrc}->{"B1"}++; } + elsif (m/- B4/){ $hashes->{rsrc}->{"B4"}++; } + elsif (m/- B2/){ $hashes->{rsrc}->{"B2"}++; } + elsif (m/- B3/){ $hashes->{rsrc}->{"B3"}++; } + elsif (m/- R1/){ $hashes->{rsrc}->{"R1"}++; } + elsif (m/- P1/){ $hashes->{rsrc}->{"P1"}++; } + elsif (m/- P1/){ $hashes->{rsrc}->{"P2"}++; } + elsif (m/- U1/){ $hashes->{rsrc}->{"U1"}++; } + else { $hashes->{rsrc}->{"other"}++; } } if ($usage =~ /g/ || $usage =~ /c/ || $usage =~ /i/ || $verb eq "yes"){ $exc = "no"; if ($_ =~ /connection from *([0-9A-fa-f\.\:]+)/i ) { - for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($1 eq $excludeIP[$xxx]){ $exc = "yes"; - writeFile($EXCOUNT,$1); + $hashes->{excount}->{$1}++; } } $ip = $1; - writeFile($IP_HASH, "$ip count"); + $hashes->{ip_hash}->{$ip}++; if ($_ =~ /conn= *([0-9]+)/i ){ if ($exc ne "yes"){ - writeFile($CONN_HASH, "$1 $ip"); + $hashes->{conn_hash}->{$1} = $ip; } } } @@ -1995,12 +1933,12 @@ sub parseLineNormal $exc = "no"; $ip = getIPfromConn($1); if ($ip eq ""){$ip = "Unknown_Host";} - for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } if ($exc ne "yes"){ - writeFile($IP_HASH, "$ip A1"); - writeFile($CONNCOUNT, "A1"); + $hashes->{A1}->{$ip}++; + $hashes->{conncount}->{"A1"}++; $connCodeCount++; } } @@ -2010,12 +1948,12 @@ sub parseLineNormal $exc = "no"; $ip = getIPfromConn($1); if ($ip eq ""){$ip = "Unknown_Host";} - for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } if ($exc ne "yes"){ - writeFile($IP_HASH, "$ip B1"); - writeFile($CONNCOUNT, "B1"); + $hashes->{B1}->{$ip}++; + $hashes->{conncount}->{"B1"}++; $connCodeCount++; } } @@ -2025,12 +1963,12 @@ sub parseLineNormal $exc = "no"; $ip = getIPfromConn($1); if ($ip eq ""){$ip = "Unknown_Host";} - for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } if ($exc ne "yes"){ - writeFile($IP_HASH, "$ip B4"); - writeFile($CONNCOUNT, "B4"); + $hashes->{B4}->{$ip}++; + $hashes->{conncount}->{"B4"}++; $connCodeCount++; } } @@ -2040,12 +1978,12 @@ sub parseLineNormal $exc = "no"; $ip = getIPfromConn($1); if ($ip eq ""){$ip = "Unknown_Host";} - for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } if ($exc ne "yes"){ - writeFile($IP_HASH, "$ip T1"); - writeFile($CONNCOUNT, "T1"); + $hashes->{T1}->{$ip}++; + $hashes->{conncount}->{"T1"}++; $connCodeCount++; } } @@ -2055,12 +1993,12 @@ sub parseLineNormal $exc = "no"; $ip = getIPfromConn($1); if ($ip eq ""){$ip = "Unknown_Host";} - for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } if ($exc ne "yes"){ - writeFile($IP_HASH, "$ip T2"); - writeFile($CONNCOUNT, "T2"); + $hashes->{T2}->{$ip}++; + $hashes->{conncount}->{"T2"}++; $connCodeCount++; } } @@ -2071,12 +2009,12 @@ sub parseLineNormal $ip = getIPfromConn($1); $maxBerSizeCount++; if ($ip eq ""){$ip = "Unknown_Host";} - for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } if ($exc ne "yes"){ - writeFile($IP_HASH, "$ip B2"); - writeFile($CONNCOUNT, "B2"); + $hashes->{B2}->{$ip}++; + $hashes->{conncount}->{"B2"}++; $connCodeCount++; } } @@ -2086,12 +2024,12 @@ sub parseLineNormal $exc = "no"; $ip = getIPfromConn($1); if ($ip eq ""){$ip = "Unknown_Host";} - for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } if ($exc ne "yes"){ - writeFile($IP_HASH, "$ip B3"); - writeFile($CONNCOUNT, "B3"); + $hashes->{B3}->{$ip}++; + $hashes->{conncount}->{"B3"}++; $connCodeCount++; } } @@ -2101,12 +2039,12 @@ sub parseLineNormal $exc = "no"; $ip = getIPfromConn($1); if ($ip eq ""){$ip = "Unknown_Host";} - for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } if ($exc ne "yes"){ - writeFile($IP_HASH, "$ip R1"); - writeFile($CONNCOUNT, "R1"); + $hashes->{R1}->{$ip}++; + $hashes->{conncount}->{"R1"}++; $connCodeCount++; } } @@ -2116,12 +2054,12 @@ sub parseLineNormal $exc = "no"; $ip = getIPfromConn($1); if ($ip eq ""){$ip = "Unknown_Host";} - for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } if ($exc ne "yes"){ - writeFile($IP_HASH, "$ip P1"); - writeFile($CONNCOUNT, "P1"); + $hashes->{P1}->{$ip}++; + $hashes->{conncount}->{"P1"}++; $connCodeCount++; } } @@ -2131,12 +2069,12 @@ sub parseLineNormal $exc = "no"; $ip = getIPfromConn($1); if ($ip eq ""){$ip = "Unknown_Host";} - for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } if ($exc ne "yes"){ - writeFile($IP_HASH, "$ip P2"); - writeFile($CONNCOUNT, "P2"); + $hashes->{P2}->{$ip}++; + $hashes->{conncount}->{"P2"}++; $connCodeCount++; } } @@ -2146,12 +2084,12 @@ sub parseLineNormal $exc = "no"; $ip = getIPfromConn($1); if ($ip eq ""){$ip = "Unknown_Host";} - for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + for (my $xxx = 0; $xxx < $#excludeIP; $xxx++){ if ($ip eq $excludeIP[$xxx]){$exc = "yes";} } if ($exc ne "yes"){ - writeFile($IP_HASH, "$ip U1"); - writeFile($CONNCOUNT, "U1"); + $hashes->{U1}->{$ip}++; + $hashes->{conncount}->{"U1"}++; $connCodeCount++; } } @@ -2162,59 +2100,60 @@ sub parseLineNormal if ($1 ne "0"){ $errorCount++;} else { $successCount++;} } - if ($_ =~ /etime= *([0-9.]+)/ ) { writeFile($ETIME, $1); inc_stats_val('etime',$1,$s_stats,$m_stats); } + if ($_ =~ /etime= *([0-9.]+)/ ) { $hashes->{etime}->{$1}++; inc_stats_val('etime',$1,$s_stats,$m_stats); } if ($_ =~ / tag=101 / || $_ =~ / tag=111 / || $_ =~ / tag=100 / || $_ =~ / tag=115 /){ - if ($_ =~ / nentries= *([0-9]+)/i ){ writeFile($NENTRIES, $1); } + if ($_ =~ / nentries= *([0-9]+)/i ){ $hashes->{nentries}->{$1}++; } } if (m/objectclass=\*/i || m/objectclass=top/i ){ if (m/ scope=2 /){ $objectclassTopCount++;} } if (m/ EXT oid=/){ $extopCount++; - if ($_ =~ /oid=\" *([0-9\.]+)/i ){ writeFile($OID,$1); } + if ($_ =~ /oid=\" *([0-9\.]+)/i ){ $hashes->{oid}->{$1}++; } if ($1 && $1 eq $startTLSoid){$startTLSCount++;} if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ writeFile($EXT_CONN, $1); } - if ($_ =~ /op= *([0-9]+)/i){ writeFile($EXT_OP, $1); } + if ($_ =~ /conn= *([0-9]+)/i){ push @{$arrays->{extconn}}, $1; } + if ($_ =~ /op= *([0-9]+)/i){ push @{$arrays->{extop}}, $1; } } } - if ($usage =~ /l/ || $verb eq "yes"){ + if (($usage =~ /l/ || $verb eq "yes") and / SRCH /){ + my ($filterConn, $filterOp); if (/ SRCH / && / attrs=/ && $_ =~ /filter=\"(.*)\" /i ){ $tmpp = $1; $tmpp =~ tr/A-Z/a-z/; $tmpp =~ s/\\22/\"/g; - writeFile($FILTER, $tmpp); - $filterVal = $tmpp; + $hashes->{filter}->{$tmpp}++; if ($_ =~ /conn= *([0-9]+)/i) { $filterConn = $1; } if ($_ =~ /op= *([0-9]+)/i) { $filterOp = $1; } } elsif (/ SRCH / && $_ =~ /filter=\"(.*)\"/i){ $tmpp = $1; $tmpp =~ tr/A-Z/a-z/; $tmpp =~ s/\\22/\"/g; - writeFile($FILTER, $tmpp); - $filterVal = $tmpp; + $hashes->{filter}->{$tmpp}++; if ($_ =~ /conn= *([0-9]+)/i) { $filterConn = $1; } if ($_ =~ /op= *([0-9]+)/i) { $filterOp = $1; } } $filterCount++; if($usage =~ /u/ || $verb eq "yes"){ - # we noly need this for the unindexed search report - writeFile($FILTERINFO, "$filterVal ,, $filterConn ,, $filterOp"); + # we only need this for the unindexed search report + push @{$arrays->{filterval}}, $tmpp; + push @{$arrays->{filterconn}}, $filterConn; + push @{$arrays->{filterop}}, $filterOp; } } if ($usage =~ /a/ || $verb eq "yes"){ if (/ SRCH / && $_ =~ /base=\"(.*)\" scope/i ){ + my ($baseConn, $baseOp, $scopeVal, $scopeConn, $scopeOp); if ($1 eq ""){ $tmpp = "Root DSE"; } else { $tmpp = $1; } $tmpp =~ tr/A-Z/a-z/; - writeFile($BASE, $tmpp); + $hashes->{base}->{$tmpp}++; # # grab the search bases & scope for potential unindexed searches # - $baseVal = $tmpp; if ($_ =~ /scope= *([0-9]+)/i) { $scopeVal = $1; } @@ -2227,9 +2166,13 @@ sub parseLineNormal $scopeOp = $1; } if($usage =~ /u/ || $verb eq "yes"){ - # we noly need this for the unindexed search report - writeFile($BASEINFO, "$baseVal ,, $baseConn ,, $baseOp"); - writeFile($SCOPEINFO, "$scopeVal ,, $scopeConn ,, $scopeOp"); + # we only need this for the unindexed search report + push @{$arrays->{baseval}}, $tmpp; + push @{$arrays->{baseconn}}, $baseConn; + push @{$arrays->{baseop}}, $baseOp; + push @{$arrays->{scopeval}}, $scopeVal; + push @{$arrays->{scopeconn}}, $scopeConn; + push @{$arrays->{scopeop}}, $scopeOp; } $baseCount++; $scopeCount++; @@ -2243,26 +2186,26 @@ sub parseLineNormal if ($usage =~ /f/ || $verb eq "yes"){ if (/ err=49 tag=/ && / dn=\"/){ if ($_ =~ /dn=\"(.*)\"/i ){ - writeFile($DS6XBADPWD, $1); + $hashes->{ds6xbadpwd}->{$1}++; } $ds6x = "true"; $badPwdCount++; } elsif (/ err=49 tag=/ ){ if ($_ =~ /conn= *([0-9]+)/i ){ - writeFile($BADPWDCONN, $1); + push @{$arrays->{badpwdconn}}, $1; $ip = getIPfromConn($1); $badPwdCount++; } if ($_ =~ /op= *([0-9]+)/i ){ - writeFile($BADPWDOP, $1); + push @{$arrays->{badpwdop}}, $1; } - writeFile($BADPWDIP, $ip); + push @{$arrays->{badpwdip}}, $ip; } } if (/ BIND / && /method=sasl/i){ $saslBindCount++; - if ($_ =~ /mech=(.*)/i ){ - writeFile($SASLMECH, $1); + if ($_ =~ /mech=(.*)/i ){ + $hashes->{saslmech}->{$1}++; } } if (/ conn=Internal op=-1 / && !/ RESULT err=/){ $internalOpCount++; } @@ -2414,7 +2357,7 @@ displayBindReport print "\nBind Report\n"; print "====================================================================\n\n"; - foreach $bindDN (sort { $bindReport{$a} <=> $bindReport{$b} } keys %bindReport) { + foreach my $bindDN (sort { $bindReport{$a} <=> $bindReport{$b} } keys %bindReport) { print("Bind DN: $bindDN\n"); print("--------------------------------------------------------------------\n"); print(" Client Addresses:\n\n"); @@ -2430,11 +2373,11 @@ displayBindReport sub printClients { - @bindConns = &cleanConns(split(' ', @_[0])); - $IPcount = "1"; + my @bindConns = &cleanConns(split(' ', $_[0])); + my $IPcount = "1"; - foreach $ip ( keys %connList ){ # Loop over all the IP addresses - foreach $bc (@bindConns){ # Loop over each bind conn number and compare it + foreach my $ip ( keys %connList ){ # Loop over all the IP addresses + foreach my $bc (@bindConns){ # Loop over each bind conn number and compare it if($connList{$ip} =~ / $bc /){ print(" [$IPcount] $ip\n"); $IPcount++; @@ -2447,22 +2390,22 @@ printClients sub cleanConns { - @dirtyConns = @_; - $#cleanConns = -1; - $c = 0; + my @dirtyConns = @_; + my @retConns = (); + my $c = 0; - for ($i = 0; $i <=$#dirtyConns; $i++){ + for (my $i = 0; $i <=$#dirtyConns; $i++){ if($dirtyConns[$i] ne ""){ - $cleanConns[$c++] = $dirtyConns[$i]; + $retConns[$c++] = $dirtyConns[$i]; } } - return @cleanConns; + return @retConns; } sub printOpStats { - $dn = @_[0]; + my $dn = $_[0]; if( $bindReport{$dn}{"failedBind"} == 0 ){ print(" Binds: " . $bindReport{$dn}{"binds"} . "\n"); @@ -2487,333 +2430,70 @@ printOpStats sub openFailed { - $open_error = @_[0]; - $file_name = @_[1]; - closeDataFiles(); + my $open_error = $_[0]; + my $file_name = $_[1]; removeDataFiles(); die ("Can not open $file_name error ($open_error)"); } sub -openDataFiles +openHashFiles { - # hash files - open ($ATTR, ">$ATTR") or do { openFailed($!, $ATTR) }; - open ($RC, ">$RC") or do { openFailed($!, $RC) }; - open ($SRC, ">$SRC") or do { openFailed($!, $SRC) }; - open ($RSRC, ">$RSRC") or do { openFailed($!, $RSRC) }; - open ($EXCOUNT, ">$EXCOUNT") or do { openFailed($!, $EXCOUNT) }; - open ($CONN_HASH, ">$CONN_HASH") or do { openFailed($!, $CONN_HASH) }; - open ($IP_HASH, ">$IP_HASH") or do { openFailed($!, $IP_HASH) }; - open ($CONNCOUNT, ">$CONNCOUNT") or do { openFailed($!, $CONNCOUNT) }; - open ($NENTRIES, ">$NENTRIES") or do { openFailed($!, $NENTRIES) }; - open ($FILTER, ">$FILTER") or do { openFailed($!, $FILTER) }; - open ($BASE, ">$BASE") or do { openFailed($!, $BASE) }; - open ($DS6XBADPWD, ">$DS6XBADPWD") or do { openFailed($!, $DS6XBADPWD) }; - open ($SASLMECH, ">$SASLMECH") or do { openFailed($!, $SASLMECH) }; - open ($BINDLIST, ">$BINDLIST") or do { openFailed($!, $BINDLIST) }; - open ($ETIME, ">$ETIME") or do { openFailed($!, $ETIME) }; - open ($OID, ">$OID") or do { openFailed($!, $OID) }; - - # array files - open($SRCH_CONN,">$SRCH_CONN") or do { openFailed($!, $SRCH_CONN) }; - open($SRCH_OP, ">$SRCH_OP") or do { openFailed($!, $SRCH_OP) }; - open($DEL_CONN, ">$DEL_CONN") or do { openFailed($!, $DEL_CONN) }; - open($DEL_OP, ">$DEL_OP") or do { openFailed($!, $DEL_OP) }; - open($MOD_CONN, ">$MOD_CONN") or do { openFailed($!, $MOD_CONN) }; - open($MOD_OP, ">$MOD_OP") or do { openFailed($!, $MOD_OP) }; - open($ADD_CONN, ">$ADD_CONN") or do { openFailed($!, $ADD_CONN) }; - open($ADD_OP, ">$ADD_OP") or do { openFailed($!, $ADD_OP) }; - open($MODRDN_CONN, ">$MODRDN_CONN") or do { openFailed($!, $MODRDN_CONN) }; - open($MODRDN_OP, ">$MODRDN_OP") or do { openFailed($!, $MODRDN_OP) }; - open($CMP_CONN, ">$CMP_CONN") or do { openFailed($!, $CMP_CONN) }; - open($CMP_OP,">$CMP_OP") or do { openFailed($!, $CMP_OP) }; - open($TARGET_CONN, ">$TARGET_CONN") or do { openFailed($!, $TARGET_CONN) }; - open($TARGET_OP, ">$TARGET_OP") or do { openFailed($!, $TARGET_OP) }; - open($MSGID, ">$MSGID") or do { openFailed($!, $MSGID) }; - open($BIND_CONN, ">$BIND_CONN") or do { openFailed($!, $BIND_CONN) }; - open($BIND_OP, ">$BIND_OP") or do { openFailed($!, $BIND_OP) }; - open($UNBIND_CONN, ">$UNBIND_CONN") or do { openFailed($!, $UNBIND_CONN) }; - open($UNBIND_OP, ">$UNBIND_OP") or do { openFailed($!, $UNBIND_OP) }; - open($EXT_CONN, ">$EXT_CONN") or do { openFailed($!, $EXT_CONN) }; - open($EXT_OP, ">$EXT_OP") or do { openFailed($!, $EXT_OP) }; - open($NOTES_A_ETIME, ">$NOTES_A_ETIME") or do { openFailed($!, $NOTES_A_ETIME) }; - open($NOTES_A_CONN, ">$NOTES_A_CONN") or do { openFailed($!, $NOTES_A_CONN) }; - open($NOTES_A_OP, ">$NOTES_A_OP") or do { openFailed($!, $NOTES_A_OP) }; - open($NOTES_A_TIME, ">$NOTES_A_TIME") or do { openFailed($!, $NOTES_A_TIME) }; - open($NOTES_A_NENTRIES, ">$NOTES_A_NENTRIES") or do { openFailed($!, $NOTES_A_NENTRIES) }; - open($NOTES_U_ETIME, ">$NOTES_U_ETIME") or do { openFailed($!, $NOTES_U_ETIME) }; - open($NOTES_U_CONN, ">$NOTES_U_CONN") or do { openFailed($!, $NOTES_U_CONN) }; - open($NOTES_U_OP, ">$NOTES_U_OP") or do { openFailed($!, $NOTES_U_OP) }; - open($NOTES_U_TIME, ">$NOTES_U_TIME") or do { openFailed($!, $NOTES_U_TIME) }; - open($NOTES_U_NENTRIES, ">$NOTES_U_NENTRIES") or do { openFailed($!, $NOTES_U_NENTRIES) }; - open($BADPWDCONN, ">$BADPWDCONN") or do { openFailed($!, $BADPWDCONN) }; - open($BADPWDOP, ">$BADPWDOP") or do { openFailed($!, $BADPWDOP) }; - open($BADPWDIP, ">$BADPWDIP") or do { openFailed($!, $NADPWDIP) }; - - # info files - open($BINDINFO, ">$BINDINFO") or do { openFailed($!, $BINDINFO) }; - open($BASEINFO, ">$BASEINFO") or do { openFailed($!, $BASEINFO) }; - open($SCOPEINFO, ">$SCOPEINFO") or do { openFailed($!, $SCOPEINFO) }; - open($FILTERINFO, ">$FILTERINFO") or do { openFailed($!, $FILTERINFO) }; -} - -sub -closeDataFiles -{ - close $ATTR; - close $RC; - close $SRC; - close $RSRC; - close $EXCOUNT; - close $CONN_HASH; - close $IP_HASH; - close $CONNCOUNT; - close $NENTRIES; - close $FILTER; - close $BASE; - close $DS6XBADPWD; - close $SASLMECH; - close $BINDLIST; - close $ETIME; - close $OID; - - # array files - close $SRCH_CONN; - close $SRCH_OP; - close $DEL_CONN; - close $DEL_OP; - close $MOD_CONN; - close $MOD_OP; - close $ADD_CONN; - close $ADD_OP; - close $MODRDN_CONN; - close $MODRDN_OP; - close $CMP_CONN; - close $CMP_OP; - close $TARGET_CONN; - close $TARGET_OP; - close $MSGID; - close $BIND_CONN; - close $BIND_OP; - close $UNBIND_CONN; - close $UNBIND_OP; - close $EXT_CONN; - close $EXT_OP; - close $NOTES_A_ETIME; - close $NOTES_A_CONN; - close $NOTES_A_OP; - close $NOTES_A_TIME; - close $NOTES_A_NENTRIES; - close $NOTES_U_ETIME; - close $NOTES_U_CONN; - close $NOTES_U_OP; - close $NOTES_U_TIME; - close $NOTES_U_NENTRIES; - close $BADPWDCONN; - close $BADPWDOP; - close $BADPWDIP; - - # info files - close $BINDINFO; - close $BASEINFO; - close $SCOPEINFO; - close $FILTERINFO; -} - -sub -removeDataFiles -{ - unlink $ATTR; - unlink $RC; - unlink $SRC; - unlink $RSRC; - unlink $EXCOUNT; - unlink $CONN_HASH; - unlink $IP_HASH; - unlink $CONNCOUNT; - unlink $NENTRIES; - unlink $FILTER; - unlink $BASE; - unlink $DS6XBADPWD; - unlink $SASLMECH; - unlink $BINDLIST; - unlink $ETIME; - unlink $OID; - - # array files - unlink $SRCH_CONN; - unlink $SRCH_OP; - unlink $DEL_CONN; - unlink $DEL_OP; - unlink $MOD_CONN; - unlink $MOD_OP; - unlink $ADD_CONN; - unlink $ADD_OP; - unlink $MODRDN_CONN; - unlink $MODRDN_OP; - unlink $CMP_CONN; - unlink $CMP_OP; - unlink $TARGET_CONN; - unlink $TARGET_OP; - unlink $MSGID; - unlink $BIND_CONN; - unlink $BIND_OP; - unlink $UNBIND_CONN; - unlink $UNBIND_OP; - unlink $EXT_CONN; - unlink $EXT_OP; - unlink $NOTES_A_ETIME; - unlink $NOTES_A_CONN; - unlink $NOTES_A_OP; - unlink $NOTES_A_TIME; - unlink $NOTES_A_NENTRIES; - unlink $NOTES_U_ETIME; - unlink $NOTES_U_CONN; - unlink $NOTES_U_OP; - unlink $NOTES_U_TIME; - unlink $NOTES_U_NENTRIES; - unlink $BADPWDCONN; - unlink $BADPWDOP; - unlink $BADPWDIP; - - # info files - unlink $BINDINFO; - unlink $BASEINFO; - unlink $SCOPEINFO; - unlink $FILTERINFO; -} - -sub -getIPfromConn -{ - $connip = @_[0]; - $retval = ""; - - close $CONN_HASH; # we can not read the file is its already open - open(CONN,"$CONN_HASH") or do { openFailed($!, $CONN_HASH) }; - while (<CONN>){ - if($_ =~ /$connip (.*)/){ - $retval = $1; - last; - } + my $dir = shift; + my %hashes = (); + for my $hn (@_) { + my %h = (); # using my in inner loop will create brand new hash every time through for tie + my $fn = "$dir/$hn.logconv.db"; + push @removefiles, $fn; + tie %h, "DB_File", $fn, O_CREAT|O_RDWR, 0600, $DB_HASH or do { openFailed($!, $fn) }; + $hashes{$hn} = \%h; } - close CONN; - #reopen file for writing(append) - open($CONN_HASH,">>$CONN_HASH") or do { openFailed($!, $CONN_HASH) }; - - return $retval; -} - -sub -writeFile -{ - $file = @_[0]; - $text = @_[1] . "\n"; - - print $file $text; + return \%hashes; } -# This hash file stores one value per line sub -getCounterHashFromFile +openArrayFiles { - $file = @_[0]; - my %hash = (); - - open(FILE,"$file") or do { openFailed($!, $file) }; - while(<FILE>){ - chomp; - $hash{$_}++; + my $dir = shift; + my %arrays = (); + for my $an (@_) { + my @ary = (); # using my in inner loop will create brand new array every time through for tie + my $fn = "$dir/$an.logconv.db"; + push @removefiles, $fn; + tie @ary, "DB_File", $fn, O_CREAT|O_RDWR, 0600, $DB_RECNO or do { openFailed($!, $fn) }; + $arrays{$an} = \@ary; } - close FILE; - - return %hash; + return \%arrays; } -# this hash file stores two values per line (2 dimension hash) sub -getTwoDimHashFromFile +removeDataFiles { - $file = @_[0]; - my %hash = (); + if (!$needCleanup) { return ; } - open(FILE,"$file") or do { openFailed($!, $file) }; - while(<FILE>){ - @parts = split (' ', $_); - chomp(@parts); - $hash{$parts[0]}{$parts[1]}++; + for my $h (keys %{$hashes}) { + untie %{$hashes->{$h}}; } - close FILE; - - return %hash; -} - -# this hash file stores two values per line (1 dimension hash) -sub -getHashFromFile -{ - $file = @_[0]; - my %hash = (); - @parts = (); - - open(FILE,"$file") or do { openFailed($!, $file ) }; - while(<FILE>){ - @parts = split (' ',$_); - chomp(@parts); - $hash{$parts[0]} = $parts[1]; + for my $a (keys %{$arrays}) { + untie @{$arrays->{$a}}; } - close FILE; - - return %hash; -} - -# Return array of values from the file -sub -getArrayFromFile -{ - my @arry; - $file = @_[0]; - $array_count = 0; - - open(FILE,"$file") or do { openFailed($!, $file) }; - while(<FILE>){ - chomp; - $arry[$array_count] = $_; - $array_count++; + for my $file (@removefiles) { + unlink $file; } - close FILE; - - return @arry; + $needCleanup = 0; } -# build the three array +END { print "Cleaning up temp files . . .\n"; removeDataFiles(); print "Done\n"; } + sub -getInfoArraysFromFile +getIPfromConn { - $file = @_[0]; - $array_count = 0; - @parts = (); - - open(FILE,"<$file") or do { openFailed($!, $file) }; - while(<FILE>){ - @parts = split (' ,, ',$_); - chomp(@parts); - if($#parts > 0){ - $fileArray1[$array_count] = $parts[0]; - $fileArray2[$array_count] = $parts[1]; - $fileArray3[$array_count] = $parts[2]; - $array_count++; - } - } - close FILE; + my $connid = shift; + return $hashes->{conn_hash}->{$connid}; } - - ####################################### # # # The End # # # ####################################### - commit fb1c15d8b89c632e0e45c885334d47b6bbb17730 Author: Thierry bordaz (tbordaz) <tbordaz(a)redhat.com&gt; Date: Fri May 17 11:48:08 2013 +0200 Ticket 47354 - Indexed search are logged with 'notes=U' in the access logs Bug Description: In the access log, the flag "notes=U" is present to warn the customer that the provided filter contains not indexed components. This flag is set independently of the fact that all entries of the database will be evaluated against the filter. So it can not be used to determine if all the entries have been scanned. Evaluated all entries is usually a performance hit for large database containing millions of entries. Fix Description: The fix consist to introduce a new flag 'notes=A' at the top level (ldbm_back_search) only when the candidate list is ALLID and all entries will be scanned. The fix also contains the update of logconv.pl to take into account this change. https://fedorahosted.org/389/ticket/47354 Reviewed by: Rich Megginson and Mark Reynolds (thanks to you !) Platforms tested: Fedora 17 Flag Day: no Doc impact: no (cherry picked from commit f2b5a975c2268b6e99bf27b45a7380d295807d81) (cherry picked from commit 15c77ad6e757625cd8a57bb5cde5e8d697bf5b25) diff --git a/ldap/admin/src/logconv.pl b/ldap/admin/src/logconv.pl index 77088ff..8ae72da 100755 --- a/ldap/admin/src/logconv.pl +++ b/ldap/admin/src/logconv.pl @@ -157,8 +157,10 @@ print "\nCommand: logconv.pl @ARGV\n\n"; $rootDNBindCount = 0; $anonymousBindCount = 0; -$unindexedSrchCount = 0; -$vlvNotesCount= 0; +$unindexedSrchCountNotesA = 0; +$unindexedSrchCountNotesU = 0; +$vlvNotesACount= 0; +$vlvNotesUCount= 0; $srchCount = 0; $fdTaken = 0; $fdReturned = 0; @@ -254,11 +256,16 @@ $UNBIND_CONN = "$dataLocation/unbindconn.logconv"; $UNBIND_OP = "$dataLocation/unbindop.logconv"; $EXT_CONN = "$dataLocation/extconn.logconv"; $EXT_OP = "$dataLocation/extop.logconv"; -$NOTES_ETIME = "$dataLocation/notesetime.logconv"; -$NOTES_CONN = "$dataLocation/notesconn.logconv"; -$NOTES_OP = "$dataLocation/notesop.logconv"; -$NOTES_TIME = "$dataLocation/notestime.logconv"; -$NOTES_NENTRIES = "$dataLocation/notesnentries.logconv"; +$NOTES_A_ETIME = "$dataLocation/notesAetime.logconv"; +$NOTES_A_CONN = "$dataLocation/notesAconn.logconv"; +$NOTES_A_OP = "$dataLocation/notesAop.logconv"; +$NOTES_A_TIME = "$dataLocation/notesAtime.logconv"; +$NOTES_A_NENTRIES = "$dataLocation/notesAnentries.logconv"; +$NOTES_U_ETIME = "$dataLocation/notesUetime.logconv"; +$NOTES_U_CONN = "$dataLocation/notesUconn.logconv"; +$NOTES_U_OP = "$dataLocation/notesUop.logconv"; +$NOTES_U_TIME = "$dataLocation/notesUtime.logconv"; +$NOTES_U_NENTRIES = "$dataLocation/notesUnentries.logconv"; $BADPWDCONN = "$dataLocation/badpwdconn.logconv"; $BADPWDOP = "$dataLocation/badpwdop.logconv"; $BADPWDIP = "$dataLocation/badpwdip.logconv"; @@ -641,20 +648,22 @@ print "Abandoned Requests: $abandonCount\n"; print "Smart Referrals Received: $referralCount\n"; print "\n"; print "VLV Operations: $vlvCount\n"; -print "VLV Unindexed Searches: $vlvNotesCount\n"; +print "VLV Unindexed Searches: $vlvNotesACount\n"; +print "VLV Unindexed Components: $vlvNotesUCount\n"; print "SORT Operations: $vlvSortCount\n"; print "\n"; print "Entire Search Base Queries: $objectclassTopCount\n"; print "Paged Searches: $pagedSearchCount\n"; -print "Unindexed Searches: $unindexedSrchCount\n"; +print "Unindexed Searches: $unindexedSrchCountNotesA\n"; +print "Unindexed Components: $unindexedSrchCountNotesU\n"; if ($verb eq "yes" || $usage =~ /u/){ - if ($unindexedSrchCount > 0){ + if ($unindexedSrchCountNotesA > 0){ %conn_hash = getHashFromFile($CONN_HASH); - @notesConn = getArrayFromFile($NOTES_CONN); - @notesOp = getArrayFromFile($NOTES_OP); - @notesEtime = getArrayFromFile($NOTES_ETIME); - @notesTime = getArrayFromFile($NOTES_TIME); - @notesNentries = getArrayFromFile($NOTES_NENTRIES); + @notesConn = getArrayFromFile($NOTES_A_CONN); + @notesOp = getArrayFromFile($NOTES_A_OP); + @notesEtime = getArrayFromFile($NOTES_A_ETIME); + @notesTime = getArrayFromFile($NOTES_A_TIME); + @notesNentries = getArrayFromFile($NOTES_A_NENTRIES); getInfoArraysFromFile($BASEINFO); @base_val = @fileArray1; @base_conn = @fileArray2; @@ -720,6 +729,78 @@ if ($verb eq "yes" || $usage =~ /u/){ undef @scope_conn; undef @scope_op; } + if ($unindexedSrchCountNotesU > 0){ + %conn_hash = getHashFromFile($CONN_HASH); + @notesConn = getArrayFromFile($NOTES_U_CONN); + @notesOp = getArrayFromFile($NOTES_U_OP); + @notesEtime = getArrayFromFile($NOTES_U_ETIME); + @notesTime = getArrayFromFile($NOTES_U_TIME); + @notesNentries = getArrayFromFile($NOTES_U_NENTRIES); + getInfoArraysFromFile($BASEINFO); + @base_val = @fileArray1; + @base_conn = @fileArray2; + @base_op = @fileArray3; + getInfoArraysFromFile($SCOPEINFO); + @scope_val = @fileArray1; + @scope_conn = @fileArray2; + @scope_op = @fileArray3; + getInfoArraysFromFile($FILTERINFO); + @filter_val = @fileArray1; + @filter_conn = @fileArray2; + @filter_op = @fileArray3; + + $notesCount = "1"; + for ($n = 0; $n <= $#notesEtime; $n++){ + @alreadyseenDN = (); + if($conn_hash{$notesConn[$n]} eq ""){ + $unindexedIp = "?"; + } else { + $unindexedIp = $conn_hash{$notesConn[$n]}; + } + print "\n Unindexed Components #".$notesCount."\n"; $notesCount++; + print " - Date/Time: $notesTime[$n]\n"; + print " - Connection Number: $notesConn[$n]\n"; + print " - Operation Number: $notesOp[$n]\n"; + print " - Etime: $notesEtime[$n]\n"; + print " - Nentries: $notesNentries[$n]\n"; + print " - IP Address: $unindexedIp\n"; + + for ($nnn = 0; $nnn < $baseCount; $nnn++){ + if ($notesConn[$n] eq $base_conn[$nnn] && $notesOp[$n] eq $base_op[$nnn]){ + print " - Search Base: $base_val[$nnn]\n"; + last; + } + } + for ($nnn = 0; $nnn < $scopeCount; $nnn++){ + if ($notesConn[$n] eq $scope_conn[$nnn] && $notesOp[$n] eq $scope_op[$nnn]){ + print " - Search Scope: $scope_val[$nnn]\n"; + last; + } + } + for ($nnn = 0; $nnn < $filterCount; $nnn++){ + if ($notesConn[$n] eq $filter_conn[$nnn] && $notesOp[$n] eq $filter_op[$nnn]){ + print " - Search Filter: $filter_val[$nnn]\n"; + last; + } + } + } + undef %conn_hash; + undef @notesConn; + undef @notesOp; + undef @notesEtime; + undef @notesTime; + undef @notesNentries; + undef @notesIp; + undef @filter_val; + undef @filter_conn; + undef @filter_op; + undef @base_val; + undef @base_conn; + undef @base_op; + undef @scope_val; + undef @scope_conn; + undef @scope_op; + } } # end of unindexed search report print "\n"; @@ -1276,10 +1357,14 @@ if ($usage =~ /j/i || $verb eq "yes"){ %conncount = getCounterHashFromFile($CONNCOUNT); print "\n----- Recommendations -----\n"; $recCount = "1"; - if ($unindexedSrchCount > 0){ + if ($unindexedSrchCountNotesA > 0){ print "\n $recCount. You have unindexed searches, this can be caused from a search on an unindexed attribute, or your returned results exceeded the allidsthreshold. Unindexed searches are not recommended. To refuse unindexed searches, switch \'nsslapd-require-index\' to \'on\' under your database entry (e.g. cn=UserRoot,cn=ldbm database,cn=plugins,cn=config).\n"; $recCount++; } + if ($unindexedSrchCountNotesU > 0){ + print "\n $recCount. You have unindexed components, this can be caused from a search on an unindexed attribute, or your returned results exceeded the allidsthreshold. Unindexed components are not recommended. To refuse unindexed searches, switch \'nsslapd-require-index\' to \'on\' under your database entry (e.g. cn=UserRoot,cn=ldbm database,cn=plugins,cn=config).\n"; + $recCount++; + } if ($conncount{"T1"} > 0){ print "\n $recCount. You have some connections that are are being closed by the idletimeout setting. You may want to increase the idletimeout if it is set low.\n"; $recCount++; @@ -1773,36 +1858,71 @@ sub parseLineNormal if (m/ RESULT err=/ && m/ notes=P/){ $pagedSearchCount++; } + if (m/ notes=A/){ + if ($_ =~ /conn= *([0-9]+)/i){ + $con = $1; + if ($_ =~ /op= *([0-9]+)/i){ $op = $1;} + } + for ($i=0; $i <= $vlvCount;$i++){ + if ($vlvconn[$i] eq $con && $vlvop[$i] eq $op){ $vlvNotesACount++; $isVlvNotes="1";} + } + if($isVlvNotes == 0){ + # We don't want to record vlv unindexed searches for our regular "bad" + # unindexed search stat, as VLV unindexed searches aren't that bad + $unindexedSrchCountNotesA++; + if($reportStats){ inc_stats('notesA',$s_stats,$m_stats); } + } + if ($usage =~ /u/ || $verb eq "yes"){ + if ($isVlvNnotes == 0 ){ + if ($_ =~ /etime= *([0-9.]+)/i ){ + writeFile($NOTES_A_ETIME, $1); + } + if ($_ =~ /conn= *([0-9]+)/i){ + writeFile($NOTES_A_CONN, $1); + } + if ($_ =~ /op= *([0-9]+)/i){ + writeFile($NOTES_A_OP, $1); + } + if ($_ =~ / *([0-9a-z:\/]+)/i){ + writeFile($NOTES_A_TIME, $1); + } + if ($_ =~ /nentries= *([0-9]+)/i ){ + writeFile($NOTES_A_NENTRIES, $1); + } + } + } + $isVlvNotes = 0; + } if (m/ notes=U/){ if ($_ =~ /conn= *([0-9]+)/i){ $con = $1; if ($_ =~ /op= *([0-9]+)/i){ $op = $1;} } for ($i=0; $i <= $vlvCount;$i++){ - if ($vlvconn[$i] eq $con && $vlvop[$i] eq $op){ $vlvNotesCount++; $isVlvNotes="1";} + if ($vlvconn[$i] eq $con && $vlvop[$i] eq $op){ $vlvNotesUCount++; $isVlvNotes="1";} } if($isVlvNotes == 0){ # We don't want to record vlv unindexed searches for our regular "bad" # unindexed search stat, as VLV unindexed searches aren't that bad - $unindexedSrchCount++; - if($reportStats){ inc_stats('notesu',$s_stats,$m_stats); } + $unindexedSrchCountNotesU++; + if($reportStats){ inc_stats('notesU',$s_stats,$m_stats); } } if ($usage =~ /u/ || $verb eq "yes"){ if ($isVlvNnotes == 0 ){ if ($_ =~ /etime= *([0-9.]+)/i ){ - writeFile($NOTES_ETIME, $1); + writeFile($NOTES_U_ETIME, $1); } if ($_ =~ /conn= *([0-9]+)/i){ - writeFile($NOTES_CONN, $1); + writeFile($NOTES_U_CONN, $1); } if ($_ =~ /op= *([0-9]+)/i){ - writeFile($NOTES_OP, $1); + writeFile($NOTES_U_OP, $1); } if ($_ =~ / *([0-9a-z:\/]+)/i){ - writeFile($NOTES_TIME, $1); + writeFile($NOTES_U_TIME, $1); } if ($_ =~ /nentries= *([0-9]+)/i ){ - writeFile($NOTES_NENTRIES, $1); + writeFile($NOTES_U_NENTRIES, $1); } } } @@ -2196,7 +2316,8 @@ reset_stats_block $stats->{'bind'}=0; $stats->{'anonbind'}=0; $stats->{'unbind'}=0; - $stats->{'notesu'}=0; + $stats->{'notesA'}=0; + $stats->{'notesU'}=0; $stats->{'etime'}=0; return; } @@ -2242,13 +2363,14 @@ print_stats_block $stats->{'bind'}, $stats->{'anonbind'}, $stats->{'unbind'}, - $stats->{'notesu'}, + $stats->{'notesA'}, + $stats->{'notesU'}, $stats->{'etime'}), "\n" ); } else { $stats->{'fh'}->print( "Time,time_t,Results,Search,Add,Mod,Modrdn,Moddn,Compare,Delete,Abandon,". - "Connections,SSL Conns,Bind,Anon Bind,Unbind,Unindexed,ElapsedTime\n" + "Connections,SSL Conns,Bind,Anon Bind,Unbind,Unindexed search,Unindexed component,ElapsedTime\n" ); } } @@ -2415,11 +2537,16 @@ openDataFiles open($UNBIND_OP, ">$UNBIND_OP") or do { openFailed($!, $UNBIND_OP) }; open($EXT_CONN, ">$EXT_CONN") or do { openFailed($!, $EXT_CONN) }; open($EXT_OP, ">$EXT_OP") or do { openFailed($!, $EXT_OP) }; - open($NOTES_ETIME, ">$NOTES_ETIME") or do { openFailed($!, $NOTES_ETIME) }; - open($NOTES_CONN, ">$NOTES_CONN") or do { openFailed($!, $NOTES_CONN) }; - open($NOTES_OP, ">$NOTES_OP") or do { openFailed($!, $NOTES_OP) }; - open($NOTES_TIME, ">$NOTES_TIME") or do { openFailed($!, $NOTES_TIME) }; - open($NOTES_NENTRIES, ">$NOTES_NENTRIES") or do { openFailed($!, $NOTES_NENTRIES) }; + open($NOTES_A_ETIME, ">$NOTES_A_ETIME") or do { openFailed($!, $NOTES_A_ETIME) }; + open($NOTES_A_CONN, ">$NOTES_A_CONN") or do { openFailed($!, $NOTES_A_CONN) }; + open($NOTES_A_OP, ">$NOTES_A_OP") or do { openFailed($!, $NOTES_A_OP) }; + open($NOTES_A_TIME, ">$NOTES_A_TIME") or do { openFailed($!, $NOTES_A_TIME) }; + open($NOTES_A_NENTRIES, ">$NOTES_A_NENTRIES") or do { openFailed($!, $NOTES_A_NENTRIES) }; + open($NOTES_U_ETIME, ">$NOTES_U_ETIME") or do { openFailed($!, $NOTES_U_ETIME) }; + open($NOTES_U_CONN, ">$NOTES_U_CONN") or do { openFailed($!, $NOTES_U_CONN) }; + open($NOTES_U_OP, ">$NOTES_U_OP") or do { openFailed($!, $NOTES_U_OP) }; + open($NOTES_U_TIME, ">$NOTES_U_TIME") or do { openFailed($!, $NOTES_U_TIME) }; + open($NOTES_U_NENTRIES, ">$NOTES_U_NENTRIES") or do { openFailed($!, $NOTES_U_NENTRIES) }; open($BADPWDCONN, ">$BADPWDCONN") or do { openFailed($!, $BADPWDCONN) }; open($BADPWDOP, ">$BADPWDOP") or do { openFailed($!, $BADPWDOP) }; open($BADPWDIP, ">$BADPWDIP") or do { openFailed($!, $NADPWDIP) }; @@ -2473,11 +2600,16 @@ closeDataFiles close $UNBIND_OP; close $EXT_CONN; close $EXT_OP; - close $NOTES_ETIME; - close $NOTES_CONN; - close $NOTES_OP; - close $NOTES_TIME; - close $NOTES_NENTRIES; + close $NOTES_A_ETIME; + close $NOTES_A_CONN; + close $NOTES_A_OP; + close $NOTES_A_TIME; + close $NOTES_A_NENTRIES; + close $NOTES_U_ETIME; + close $NOTES_U_CONN; + close $NOTES_U_OP; + close $NOTES_U_TIME; + close $NOTES_U_NENTRIES; close $BADPWDCONN; close $BADPWDOP; close $BADPWDIP; @@ -2531,11 +2663,16 @@ removeDataFiles unlink $UNBIND_OP; unlink $EXT_CONN; unlink $EXT_OP; - unlink $NOTES_ETIME; - unlink $NOTES_CONN; - unlink $NOTES_OP; - unlink $NOTES_TIME; - unlink $NOTES_NENTRIES; + unlink $NOTES_A_ETIME; + unlink $NOTES_A_CONN; + unlink $NOTES_A_OP; + unlink $NOTES_A_TIME; + unlink $NOTES_A_NENTRIES; + unlink $NOTES_U_ETIME; + unlink $NOTES_U_CONN; + unlink $NOTES_U_OP; + unlink $NOTES_U_TIME; + unlink $NOTES_U_NENTRIES; unlink $BADPWDCONN; unlink $BADPWDOP; unlink $BADPWDIP; diff --git a/ldap/servers/slapd/back-ldbm/ldbm_search.c b/ldap/servers/slapd/back-ldbm/ldbm_search.c index 76b41d5..f2f92b0 100644 --- a/ldap/servers/slapd/back-ldbm/ldbm_search.c +++ b/ldap/servers/slapd/back-ldbm/ldbm_search.c @@ -830,7 +830,7 @@ ldbm_back_search( Slapi_PBlock *pb ) * to record that fact. */ if ( NULL != candidates && ALLIDS( candidates )) { - unsigned int opnote = SLAPI_OP_NOTE_UNINDEXED; + unsigned int opnote; int ri = 0; int pr_idx = -1; @@ -853,6 +853,10 @@ ldbm_back_search( Slapi_PBlock *pb ) } } + slapi_pblock_get( pb, SLAPI_OPERATION_NOTES, &opnote ); + opnote |= SLAPI_OP_NOTE_FULL_UNINDEXED; /* the full filter leads to an unindexed search */ + opnote &= ~SLAPI_OP_NOTE_UNINDEXED; /* this note is useless because FULL_UNINDEXED includes UNINDEXED */ + slapi_pblock_set( pb, SLAPI_OPERATION_NOTES, NULL ); slapi_pblock_set( pb, SLAPI_OPERATION_NOTES, &opnote ); slapi_pblock_get( pb, SLAPI_PAGED_RESULTS_INDEX, &pr_idx ); pagedresults_set_unindexed( pb->pb_conn, pb->pb_op, pr_idx ); diff --git a/ldap/servers/slapd/result.c b/ldap/servers/slapd/result.c index e124d0b..8f64dd7 100644 --- a/ldap/servers/slapd/result.c +++ b/ldap/servers/slapd/result.c @@ -1617,7 +1617,8 @@ struct slapi_note_map { static struct slapi_note_map notemap[] = { { SLAPI_OP_NOTE_UNINDEXED, "U" }, - { SLAPI_OP_NOTE_SIMPLEPAGED, "P" } + { SLAPI_OP_NOTE_SIMPLEPAGED, "P" }, + { SLAPI_OP_NOTE_FULL_UNINDEXED, "A" } }; #define SLAPI_NOTEMAP_COUNT ( sizeof(notemap) / sizeof(struct slapi_note_map)) diff --git a/ldap/servers/slapd/slapi-plugin.h b/ldap/servers/slapd/slapi-plugin.h index 957e4ff..3583f77 100644 --- a/ldap/servers/slapd/slapi-plugin.h +++ b/ldap/servers/slapd/slapi-plugin.h @@ -6694,6 +6694,7 @@ typedef struct slapi_plugindesc { #define SLAPI_OPERATION_NOTES 57 #define SLAPI_OP_NOTE_UNINDEXED 0x01 #define SLAPI_OP_NOTE_SIMPLEPAGED 0x02 +#define SLAPI_OP_NOTE_FULL_UNINDEXED 0x04 /* Allows controls to be passed before operation object is created */ #define SLAPI_CONTROLS_ARG 58 commit ba61cebeb12b4d11706a80f746f2fd97f5b5f7f9 Author: Mark Reynolds <mreynolds(a)redhat.com&gt; Date: Wed Aug 14 15:32:36 2013 -0400 Ticket 47461 - logconv.pl - Use of comma-less variable list is deprecated Description: In newer versions of perl commas are needed to separate variables when "formatting" output. Also fixed a typo. https://fedorahosted.org/389/ticket/47461 Reviewed by: nhosoi(Thanks!) (cherry picked from commit 7e9cae5fa2292168e88e87eee2f9249171a1e9d8) (cherry picked from commit 3e7ee7ca886feb56b3b26df97882c40d81bfff55) diff --git a/ldap/admin/src/logconv.pl b/ldap/admin/src/logconv.pl index efc5970..77088ff 100755 --- a/ldap/admin/src/logconv.pl +++ b/ldap/admin/src/logconv.pl @@ -594,7 +594,7 @@ print "Restarts: $serverRestartCount\n"; print "Total Connections: $connectionCount\n"; print " - StartTLS Connections: $startTLSCount\n"; print " - LDAPS Connections: $sslCount\n"; -print " - LDAPI Conections: $ldapiCount\n"; +print " - LDAPI Connections: $ldapiCount\n"; print "Peak Concurrent Connections: $maxsimConnection\n"; print "Total Operations: $allOps\n"; print "Total Results: $allResults\n"; @@ -614,20 +614,20 @@ $compareStat = sprintf "(%.2f/sec) (%.2f/min)\n",$cmpCount/$totalTimeInSecs, $c $bindCountStat = sprintf "(%.2f/sec) (%.2f/min)\n",$bindCount/$totalTimeInSecs, $bindCount/($totalTimeInSecs/60); format STDOUT = -Searches: @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< - $srchCount, $searchStat -Modifications: @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< - $modCount, $modStat -Adds: @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< - $addCount, $addStat -Deletes: @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< - $delCount, $deleteStat -Mod RDNs: @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< - $modrdnCount, $modrdnStat -Compares: @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< - $cmpCount, $compareStat -Binds: @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< - $bindCount $bindCountStat +Searches: @<<<<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + $srchCount, $searchStat, +Modifications: @<<<<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + $modCount, $modStat, +Adds: @<<<<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + $addCount, $addStat, +Deletes: @<<<<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + $delCount, $deleteStat, +Mod RDNs: @<<<<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + $modrdnCount, $modrdnStat, +Compares: @<<<<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + $cmpCount, $compareStat, +Binds: @<<<<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + $bindCount, $bindCountStat, . write STDOUT; commit 65eb63b8a35afe5026386b1dda6eb321f52e18f2 Author: Mark Reynolds <mreynolds(a)redhat.com&gt; Date: Mon Jul 29 15:44:10 2013 -0400 Ticket 47447 - logconv.pl man page missing -m,-M,-B,-D Fix Description: Added the missing info, and also updated the main arguments with their long option names. https://fedorahosted.org/389/ticket/47447 Reviewed by: nhosoi(Thanks!) (cherry picked from commit 4ea4c8bf91f5c28bd643b0c06f590ec164156358) (cherry picked from commit ed245c45813255400c6962707f7269caa253e953) diff --git a/man/man1/logconv.pl.1 b/man/man1/logconv.pl.1 index ab81c69..c2a8cd8 100644 --- a/man/man1/logconv.pl.1 +++ b/man/man1/logconv.pl.1 @@ -29,30 +29,46 @@ line .SH OPTIONS A summary of options is included below: .TP -.B \fB\-h\fR +.B \fB\-h, \-\-help\fR help/usage .TP -.B \fB\-d\fR <Directory Managers DN> +.B \fB\-d, \-\-rootDN\fR <Directory Managers DN> DEFAULT \-> cn=directory manager .TP -.B \fB\-s\fR <Number of results to return per category> +.B \fB\-D, \-\-data\fR <Location for temporary data files> +DEFAULT \-> /tmp +.br +TIP \-> If there is not enough RAM, a RAM disk can be used instead: + + mkdir /dev/shm/logconv, and use this directory for the "-D" value. +.TP +.B \fB\-s, \-\-sizeLimit\fR <Number of results to return per category> DEFAULT \-> 20 .TP -.B \fB\-X\fR <IP address to exclude from connection stats> +.B \fB\-X, \-\-excludeIP\fR <IP address to exclude from connection stats> E.g. Load balancers .TP -.B \fB\-v\fR show version of tool +.B \fB\-v, \-\-version\fR show version of tool Print version of the tool .TP -.B \fB\-S\fR <time to begin analyzing logfile from> +.B \fB\-S, \-\-startTime\fR <time to begin analyzing logfile from> Time to begin analyzing logfile from -E.g. [28/Mar/2002:13:14:22 \fB\-0800]\fR +E.g. [28/Mar/2002:13:14:22 \f \-0800]\fR .TP -.B \fB\-E\fR <time to stop analyzing logfile> +.B \fB\-E, \-\-endTime\fR <time to stop analyzing logfile> Time to stop analyzing logfile from -E.g. [28/Mar/2002:13:24:62 \fB\-0800]\fR +E.g. [28/Mar/2002:13:24:62 \f \-0800]\fR +.TP +.B \fB\-M, \-\-reportFileMins\fR <CSV output file> +This option creates a CSV report for spreadsheets. +.TP +.B \fB\-m, \-\-reportFileSecs\fR <CSV output file> +This option creates a CSV report for spreadsheets. +.TP +.B \fB\-B, \-\-bind\fR <ALL | ANONYMOUS | "Bind DN"> +This generates a report based on either ALL bind dn's, anonymous binds, or a specific DN. .TP -\fB\-V\fR <enable verbose output \- includes all stats listed below> +\fB\-V, \-\-verbose\fR <enable verbose output \- includes all stats listed below> Verbose output .TP .B \fB\-[efcibaltnxgju]\fR commit 02a70f7e8f3bf5deac9039b51c31a206e628e9e1 Author: Rich Megginson <rmeggins(a)redhat.com&gt; Date: Wed Apr 24 11:44:40 2013 -0600 Ticket #47348 - add etimes to per second/minute stats https://fedorahosted.org/389/ticket/47348 Reviewed by: mreynolds (Thanks!) Branch: 389-ds-base-1.3.1 Fix Description: Add an "ElapsedTime" column to the -m/-M output. This column is the cumulative etimes of all operations during that time period. Platforms tested: RHEL6 x86_64 Flag Day: no Doc impact: yes (cherry picked from commit 9813b219ad2aa7b7194d9295dd529e12889a8534) Ticket #47349 - DS instance crashes under a high load https://fedorahosted.org/389/ticket/47349 Reviewed by: nkinder (Thanks!) Branch: 389-ds-base-1.3.1 Fix Description: handle_new_connection initializes the connection object, then calls connection_table_move_connection_on_to_active_list to put it on the list of active connections, then unlocks the c_mutex, then calls connection_new_private to allocate c_private. If another thread interrupts after the conn has been moved to the active list, but before c_private has been allocated, the new conn will be available via connection_table_iterate_active_connections where table_iterate_function will attempt to dereference the NULL c_private. The fix is to move connection_new_private inside the c_mutex lock, and to move connection_table_move_connection_on_to_active_list to be the very last thing before releasing the c_mutex lock. Once the conn is on the active list it is live and we cannot do anything else to it. Note: I have still not been able to reproduce the problem in a non-debug optimized build. Platforms tested: RHEL6 x86_64 Note: Before patch, server would crash within 5 minutes. After patch, server has been running for several days in customer environment. Flag Day: no Doc impact: no (cherry picked from commit 05d209432571dc64b242ae47113ae4cbb43607d2) (cherry picked from commit c7fc203ffe09809e430ff1217ec352db95d636a0) diff --git a/ldap/admin/src/logconv.pl b/ldap/admin/src/logconv.pl index 3b8adc5..efc5970 100755 --- a/ldap/admin/src/logconv.pl +++ b/ldap/admin/src/logconv.pl @@ -2042,7 +2042,7 @@ sub parseLineNormal if ($1 ne "0"){ $errorCount++;} else { $successCount++;} } - if ($_ =~ /etime= *([0-9.]+)/ ) { writeFile($ETIME, $1);} + if ($_ =~ /etime= *([0-9.]+)/ ) { writeFile($ETIME, $1); inc_stats_val('etime',$1,$s_stats,$m_stats); } if ($_ =~ / tag=101 / || $_ =~ / tag=111 / || $_ =~ / tag=100 / || $_ =~ / tag=115 /){ if ($_ =~ / nentries= *([0-9]+)/i ){ writeFile($NENTRIES, $1); } } @@ -2197,6 +2197,7 @@ reset_stats_block $stats->{'anonbind'}=0; $stats->{'unbind'}=0; $stats->{'notesu'}=0; + $stats->{'etime'}=0; return; } @@ -2241,12 +2242,13 @@ print_stats_block $stats->{'bind'}, $stats->{'anonbind'}, $stats->{'unbind'}, - $stats->{'notesu'} ), + $stats->{'notesu'}, + $stats->{'etime'}), "\n" ); } else { $stats->{'fh'}->print( "Time,time_t,Results,Search,Add,Mod,Modrdn,Moddn,Compare,Delete,Abandon,". - "Connections,SSL Conns,Bind,Anon Bind,Unbind,Unindexed\n" + "Connections,SSL Conns,Bind,Anon Bind,Unbind,Unindexed,ElapsedTime\n" ); } } @@ -2265,6 +2267,20 @@ inc_stats return; } +# like inc_stats, but increments the block counter with the given value e.g. +# 'statname1', val, statblock1, statblock2, ... +sub +inc_stats_val +{ + my $n = shift; + my $val = shift; + foreach(@_){ + $_->{$n} += $val + if exists $_->{$n}; + } + return; +} + sub displayBindReport { commit eea06437c2be14fa9ab70dc983e2ab138f8d62c8 Author: Rich Megginson <rmeggins(a)redhat.com&gt; Date: Mon Apr 22 20:28:14 2013 -0600 Ticket #47341 - logconv.pl -m time calculation is wrong https://fedorahosted.org/389/ticket/47341 Reviewed by: nkinder (Thanks!) Branch: 389-ds-base-1.3.1 Fix Description: The variable name is $hr, not $hours. The regex for the sign in the timezone should have been (.) not (?). Platforms tested: RHEL6 x86_64 Flag Day: no Doc impact: no (cherry picked from commit 87f5ef5a97437cc92afe2496cd06e5d2ec71d9ff) (cherry picked from commit 1878bbfef7ae76ed18d709d5414e21cbd0961a38) diff --git a/ldap/admin/src/logconv.pl b/ldap/admin/src/logconv.pl index 757f799..3b8adc5 100755 --- a/ldap/admin/src/logconv.pl +++ b/ldap/admin/src/logconv.pl @@ -1574,7 +1574,7 @@ sub parseLineNormal { # tz offset change $lastzone=$tzone; - ($sign,$hr,$min) = $tzone =~ m/(?)(\d\d)(\d\d)/; + ($sign,$hr,$min) = $tzone =~ m/(.)(\d\d)(\d\d)/; $tzoff = $hr*3600 + $min*60; $tzoff *= -1 if $sign eq '-'; @@ -1582,7 +1582,7 @@ sub parseLineNormal } ($date, $hr, $min, $sec) = split (':', $time); ($day, $mon, $yr) = split ('/', $date); - $newmin = timegm(0, $min, $hours, $day, $monthname{$mon}, $yr) - $tzoff; + $newmin = timegm(0, $min, $hr, $day, $monthname{$mon}, $yr) - $tzoff; $gmtime = $newmin + $sec; print_stats_block( $s_stats ); reset_stats_block( $s_stats, $gmtime, $time.' '.$tzone ); commit 54193cb97b9157d0da785719a3d91f0f58cfb5e7 Author: Rich Megginson <rmeggins(a)redhat.com&gt; Date: Fri Apr 19 14:55:41 2013 -0600 Ticket #47336 - logconv.pl -m not working for all stats https://fedorahosted.org/389/ticket/47336 Reviewed by: nhosoi (Thanks!) Branch: 389-ds-base-1.3.1 Fix Description: Code was using $reportStat instead of $reportStats. Also, some fields such as connections, ssl connections were not being kept. Finally, some headers were missing. Platforms tested: RHEL6 x86_64 Flag Day: no Doc impact: no (cherry picked from commit 27f2595bfd54c85bb169ac45cac2cdbfd4de17fc) (cherry picked from commit 26bd66a10ab942c8782294d89a9e3e1768bc7936) diff --git a/ldap/admin/src/logconv.pl b/ldap/admin/src/logconv.pl index 7e0bfd8..757f799 100755 --- a/ldap/admin/src/logconv.pl +++ b/ldap/admin/src/logconv.pl @@ -1564,7 +1564,7 @@ sub parseLineNormal # Additional performance stats ($time, $tzone) = split (' ', $_); - if ($reportStat && $time ne $last_tm) + if ($reportStats && $time ne $last_tm) { $last_tm = $time; $time =~ s/\[//; @@ -1597,11 +1597,11 @@ sub parseLineNormal if (m/ RESULT err/){ $allResults++; - if($reportStat){ inc_stats('results',$s_stats,$m_stats); } + if($reportStats){ inc_stats('results',$s_stats,$m_stats); } } if (m/ SRCH/){ $srchCount++; - if($reportStat){ inc_stats('srch',$s_stats,$m_stats); } + if($reportStats){ inc_stats('srch',$s_stats,$m_stats); } if ($_ =~ / attrs=\"(.*)\"/i){ $anyAttrs++; $attrs = $1 . " "; @@ -1620,7 +1620,7 @@ sub parseLineNormal } if (m/ DEL/){ $delCount++; - if($reportStat){ inc_stats('del',$s_stats,$m_stats); } + if($reportStats){ inc_stats('del',$s_stats,$m_stats); } if ($verb eq "yes"){ if ($_ =~ /conn= *([0-9]+)/i){ writeFile($DEL_CONN, $1);} if ($_ =~ /op= *([0-9]+)/i){ writeFile($DEL_OP, $1);} @@ -1628,7 +1628,7 @@ sub parseLineNormal } if (m/ MOD dn=/){ $modCount++; - if($reportStat){ inc_stats('mod',$s_stats,$m_stats); } + if($reportStats){ inc_stats('mod',$s_stats,$m_stats); } if ($verb eq "yes"){ if ($_ =~ /conn= *([0-9]+)/i){ writeFile($MOD_CONN, $1);} if ($_ =~ /op= *([0-9]+)/i){ writeFile($MOD_OP, $1); } @@ -1636,7 +1636,7 @@ sub parseLineNormal } if (m/ ADD/){ $addCount++; - if($reportStat){ inc_stats('add',$s_stats,$m_stats); } + if($reportStats){ inc_stats('add',$s_stats,$m_stats); } if ($verb eq "yes"){ if ($_ =~ /conn= *([0-9]+)/i){ writeFile($ADD_CONN, $1); } if ($_ =~ /op= *([0-9]+)/i){ writeFile($ADD_OP, $1); } @@ -1644,7 +1644,7 @@ sub parseLineNormal } if (m/ MODRDN/){ $modrdnCount++; - if($reportStat){ inc_stats('modrdn',$s_stats,$m_stats); } + if($reportStats){ inc_stats('modrdn',$s_stats,$m_stats); } if ($verb eq "yes"){ if ($_ =~ /conn= *([0-9]+)/i){ writeFile($MODRDN_CONN, $1); } if ($_ =~ /op= *([0-9]+)/i){ writeFile($MODRDN_OP, $1); } @@ -1652,7 +1652,7 @@ sub parseLineNormal } if (m/ CMP dn=/){ $cmpCount++; - if($reportStat){ inc_stats('cmp',$s_stats,$m_stats); } + if($reportStats){ inc_stats('cmp',$s_stats,$m_stats); } if ($verb eq "yes" || $usage =~ /g/i){ if ($_ =~ /conn= *([0-9]+)/i){ writeFile($CMP_CONN, $1);} if ($_ =~ /op= *([0-9]+)/i){ writeFile($CMP_OP, $1);} @@ -1660,7 +1660,7 @@ sub parseLineNormal } if (m/ ABANDON /){ $abandonCount++; - if($reportStat){ inc_stats('abandon',$s_stats,$m_stats); } + if($reportStats){ inc_stats('abandon',$s_stats,$m_stats); } $allResults++; if ($_ =~ /targetop= *([0-9a-zA-Z]+)/i ){ writeFile($TARGET_OP, $1); @@ -1680,12 +1680,12 @@ sub parseLineNormal if (m/ version=2/){$v2BindCount++} if (m/ version=3/){$v3BindCount++} if (m/ conn=1 fd=/){$serverRestartCount++} - if (m/ SSL connection from/){$sslCount++;} + if (m/ SSL connection from/){$sslCount++; if($reportStats){ inc_stats('sslconns',$s_stats,$m_stats); }} if (m/ connection from local to /){$ldapiCount++;} if($_ =~ /AUTOBIND dn=\"(.*)\"/){ $autobindCount++; $bindCount++; - if($reportStat){ inc_stats('bind',$s_stats,$m_stats); } + if($reportStats){ inc_stats('bind',$s_stats,$m_stats); } if ($1 ne ""){ $tmpp = $1; $tmpp =~ tr/A-Z/a-z/; @@ -1705,7 +1705,10 @@ sub parseLineNormal for ($xxx =0; $xxx <= $#excludeIP; $xxx++){ if ($excludeIP[$xxx] eq $1){$exc = "yes";} } - if ($exc ne "yes"){ $connectionCount++;} + if ($exc ne "yes"){ + $connectionCount++; + if($reportStats){ inc_stats('conns',$s_stats,$m_stats); } + } } $simConnection++; if ($simConnection > $maxsimConnection) { @@ -1740,7 +1743,7 @@ sub parseLineNormal if ($diff >= 16) { $latency[6] ++;} } if (m/ BIND/ && $_ =~ /dn=\"(.*)\" method/i ){ - if($reportStat){ inc_stats('bind',$s_stats,$m_stats); } + if($reportStats){ inc_stats('bind',$s_stats,$m_stats); } $bindCount++; if ($1 ne ""){ if($1 eq $rootDN){$rootDNBindCount++;} @@ -1782,7 +1785,7 @@ sub parseLineNormal # We don't want to record vlv unindexed searches for our regular "bad" # unindexed search stat, as VLV unindexed searches aren't that bad $unindexedSrchCount++; - if($reportStat){ inc_stats('notesu',$s_stats,$m_stats); } + if($reportStats){ inc_stats('notesu',$s_stats,$m_stats); } } if ($usage =~ /u/ || $verb eq "yes"){ if ($isVlvNnotes == 0 ){ @@ -2242,7 +2245,7 @@ print_stats_block "\n" ); } else { $stats->{'fh'}->print( - "Time,time_t,Results,Search,Add,Mod,Modrdn,Delete,Abandon,". + "Time,time_t,Results,Search,Add,Mod,Modrdn,Moddn,Compare,Delete,Abandon,". "Connections,SSL Conns,Bind,Anon Bind,Unbind,Unindexed\n" ); } commit 9f382458ee799e08f9ceddab73111011ea15e3a4 Author: Mark Reynolds <mreynolds(a)redhat.com&gt; Date: Thu Mar 21 16:12:41 2013 -0400 Ticket 611 - logconv.pl missing stats for StartTLS, LDAPI, and AUTOBIND RFE Description: Previously there were no stats for StartTLS, LDAPI, and AUTOTBIND connections/binds. https://fedorahosted.org/389/ticket/611 Reviewed by: richm(Thanks!) (cherry picked from commit 59ae73c25c316c81c1c18ea702c8e793c922e1d8) diff --git a/ldap/admin/src/logconv.pl b/ldap/admin/src/logconv.pl index 7ea7f09..7e0bfd8 100755 --- a/ldap/admin/src/logconv.pl +++ b/ldap/admin/src/logconv.pl @@ -70,6 +70,7 @@ $endFlag = 0; $endTime = 0; $reportStats = ""; $dataLocation = "/tmp"; +$startTLSoid = "1.3.6.1.4.1.1466.20037"; $s_stats = new_stats_block( ); $m_stats = new_stats_block( ); @@ -208,6 +209,9 @@ $maxsimConnection = 0; $firstFile = 1; $elapsedDays = 0; $logCount = 0; +$startTLSCount = 0; +$ldapiCount = 0; +$autobindCount = 0; $limit = 25000; # number of lines processed to trigger output # hash files @@ -588,7 +592,9 @@ if($reportBinds eq "yes"){ print "Restarts: $serverRestartCount\n"; print "Total Connections: $connectionCount\n"; -print "SSL Connections: $sslCount\n"; +print " - StartTLS Connections: $startTLSCount\n"; +print " - LDAPS Connections: $sslCount\n"; +print " - LDAPI Conections: $ldapiCount\n"; print "Peak Concurrent Connections: $maxsimConnection\n"; print "Total Operations: $allOps\n"; print "Total Results: $allResults\n"; @@ -760,21 +766,22 @@ print "Max BER Size Exceeded: $maxBerSizeCount\n"; print "\n"; print "Binds: $bindCount\n"; print "Unbinds: $unbindCount\n"; -print "\n LDAP v2 Binds: $v2BindCount\n"; -print " LDAP v3 Binds: $v3BindCount\n"; -print " SSL Client Binds: $sslClientBindCount\n"; -print " Failed SSL Client Binds: $sslClientFailedCount\n"; -print " SASL Binds: $saslBindCount\n"; +print " - LDAP v2 Binds: $v2BindCount\n"; +print " - LDAP v3 Binds: $v3BindCount\n"; +print " - AUTOBINDs: $autobindCount\n"; +print " - SSL Client Binds: $sslClientBindCount\n"; +print " - Failed SSL Client Binds: $sslClientFailedCount\n"; +print " - SASL Binds: $saslBindCount\n"; if ($saslBindCount > 0){ foreach $saslb ( sort {$saslmech{$b} <=> $saslmech{$a} } (keys %saslmech) ){ - printf " %-4s %-12s\n",$saslmech{$saslb}, $saslb; + printf " %-4s %-12s\n",$saslmech{$saslb}, $saslb; } } -print "\n Directory Manager Binds: $rootDNBindCount\n"; -print " Anonymous Binds: $anonymousBindCount\n"; +print " - Directory Manager Binds: $rootDNBindCount\n"; +print " - Anonymous Binds: $anonymousBindCount\n"; $otherBindCount = $bindCount -($rootDNBindCount + $anonymousBindCount); -print " Other Binds: $otherBindCount\n\n"; +print " - Other Binds: $otherBindCount\n\n"; ########################################################################## # Verbose Logging Section # @@ -1674,6 +1681,24 @@ sub parseLineNormal if (m/ version=3/){$v3BindCount++} if (m/ conn=1 fd=/){$serverRestartCount++} if (m/ SSL connection from/){$sslCount++;} + if (m/ connection from local to /){$ldapiCount++;} + if($_ =~ /AUTOBIND dn=\"(.*)\"/){ + $autobindCount++; + $bindCount++; + if($reportStat){ inc_stats('bind',$s_stats,$m_stats); } + if ($1 ne ""){ + $tmpp = $1; + $tmpp =~ tr/A-Z/a-z/; + writeFile($BINDLIST, $tmpp); + if($1 eq $rootDN){ + $rootDNBindCount++; + } + } else { + $anonymousBindCount++; + writeFile($BINDLIST, "Anonymous Binds"); + inc_stats('anonbind',$s_stats,$m_stats); + } + } if (m/ connection from/){ $exc = "no"; if ($_ =~ /connection from *([0-9A-Fa-f\.\:]+)/i ){ @@ -1718,6 +1743,7 @@ sub parseLineNormal if($reportStat){ inc_stats('bind',$s_stats,$m_stats); } $bindCount++; if ($1 ne ""){ + if($1 eq $rootDN){$rootDNBindCount++;} $tmpp = $1; $tmpp =~ tr/A-Z/a-z/; writeFile($BINDLIST, $tmpp); @@ -1728,9 +1754,6 @@ sub parseLineNormal # only need this for the failed bind report writeFile($BINDINFO, "$bindVal ,, $bindConn ,, $bindOp"); } - if($1 eq $rootDN){ - $rootDNBindCount++; - } } else { $anonymousBindCount++; writeFile($BINDLIST, "Anonymous Binds"); @@ -2026,6 +2049,7 @@ sub parseLineNormal if (m/ EXT oid=/){ $extopCount++; if ($_ =~ /oid=\" *([0-9\.]+)/i ){ writeFile($OID,$1); } + if ($1 && $1 eq $startTLSoid){$startTLSCount++;} if ($verb eq "yes"){ if ($_ =~ /conn= *([0-9]+)/i){ writeFile($EXT_CONN, $1); } if ($_ =~ /op= *([0-9]+)/i){ writeFile($EXT_OP, $1); } commit 9866682da7fb35e16922d94404c4265701fc797a Author: Mark Reynolds <mreynolds(a)redhat.com&gt; Date: Tue Jan 15 15:36:33 2013 -0500 Ticket 419 - logconv.pl - improve memory management Bug Description: logconv.pl memory usage is very high, and doesn't appear to be efficient. Fix Description: For verbose stats/hashes/arrays we now write this information to disk, and read it in and process it when printing the report. This dramatically improved memory growth when dealing with larger and larger log sets. Also updated the oids which some were incorrect, but many new oids were missing. Added stats for "Paged Searches", and "Max BER size exceeded". Finally, renamed most variables to be readable names, removed redundant counters, and other generic imrpovements. https://fedorahosted.org/389/ticket/419 Reviewed by: Noriko(Thanks!) (cherry picked from commit 16651d642bf6f7ba624f62cac867b495253b4f3f) diff --git a/ldap/admin/src/logconv.pl b/ldap/admin/src/logconv.pl new file mode 100755 index 0000000..7ea7f09 --- /dev/null +++ b/ldap/admin/src/logconv.pl @@ -0,0 +1,2639 @@ +#!/usr/bin/env perl + +# +# BEGIN COPYRIGHT BLOCK +# This Program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation; version 2 of the License. +# +# This Program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# this Program; if not, write to the Free Software Foundation, Inc., 59 Temple +# Place, Suite 330, Boston, MA 02111-1307 USA. +# +# In addition, as a special exception, Red Hat, Inc. gives You the additional +# right to link the code of this Program with code not covered under the GNU +# General Public License ("Non-GPL Code") and to distribute linked combinations +# including the two, subject to the limitations in this paragraph. Non-GPL Code +# permitted under this exception must only link to the code of this Program +# through those well defined interfaces identified in the file named EXCEPTION +# found in the source code files (the "Approved Interfaces"). The files of +# Non-GPL Code may instantiate templates or use macros or inline functions from +# the Approved Interfaces without causing the resulting work to be covered by +# the GNU General Public License. Only Red Hat, Inc. may make changes or +# additions to the list of Approved Interfaces. You must obey the GNU General +# Public License in all respects for all of the Program code and other code used +# in conjunction with the Program except the Non-GPL Code covered by this +# exception. If you modify this file, you may extend this exception to your +# version of the file, but you are not obligated to do so. If you do not wish to +# provide this exception without modification, you must delete this exception +# statement from your version and license this file solely under the GPL without +# exception. +# +# +# Copyright (C) 2001 Sun Microsystems, Inc. Used by permission. +# Copyright (C) 2013 Red Hat, Inc. +# All rights reserved. +# END COPYRIGHT BLOCK +# + +# +# Check for usage +# +use Time::Local; +use IO::File; +use Getopt::Long; + +Getopt::Long::Configure ("bundling"); +Getopt::Long::Configure ("permute"); + +if ($#ARGV < 0){; +&displayUsage; +} + +####################################### +# # +# parse commandline switches # +# # +####################################### + +$file_count = 0; +$arg_count = 0; +$logversion = "7.0"; +$sizeCount = "20"; +$startFlag = 0; +$startTime = 0; +$endFlag = 0; +$endTime = 0; +$reportStats = ""; +$dataLocation = "/tmp"; +$s_stats = new_stats_block( ); +$m_stats = new_stats_block( ); + +GetOptions( + 'd|rootDN=s' => \$rootDN, + 'v|version' => sub { print "Access Log Analyzer v$logversion\n"; exit (0); }, + 'V|verbose' => sub { $verb = "yes"; }, + 'D|data=s' => \$dataLocation, + 'X|excludeIP=s' => \$excludeIP[$xi++], + 's|sizeLimit=s' => \$sizeCount, + 'S|startTime=s' => \$startTime, + 'E|endTime=s' => \$endTime, + 'B|bind=s' => sub { $reportBinds = "yes"; $bindReportDN=($_[1]) }, + 'm|reportFileSecs=s' => sub { my ($opt,$value) = @_; $s_stats = new_stats_block($value); $reportStats = "-m";}, + 'M|reportFileMins=s' => sub { my ($opt,$value) = @_; $m_stats = new_stats_block($value); $reportStats = "-M";}, + 'h|help' => sub { displayUsage() }, + # usage options '-efcibaltnxgjuiryp' + 'e' => sub { $usage = $usage . "e"; }, + 'f' => sub { $usage = $usage . "f"; }, + 'c' => sub { $usage = $usage . "c"; }, + 'i' => sub { $usage = $usage . "i"; }, + 'b' => sub { $usage = $usage . "b"; }, + 'a' => sub { $usage = $usage . "a"; }, + 'l' => sub { $usage = $usage . "l"; }, + 't' => sub { $usage = $usage . "t"; }, + 'n' => sub { $usage = $usage . "n"; }, + 'x' => sub { $usage = $usage . "x"; }, + 'g' => sub { $usage = $usage . "g"; }, + 'j' => sub { $usage = $usage . "j"; }, + 'u' => sub { $usage = $usage . "u"; }, + 'r' => sub { $usage = $usage . "r"; }, + 'y' => sub { $usage = $usage . "y"; }, + 'p' => sub { $usage = $usage . "p"; } +); + +# +# setup the report Bind DN if any +# +if($reportBinds eq "yes"){ + $bindReportDN =~ tr/A-Z/a-z/; + if($bindReportDN eq "all"){ + $bindReportDN = ""; + } + if($bindReportDN eq "anonymous"){ + $bindReportDN = "Anonymous"; + } +} + +# +# set the default root DN +# +if($rootDN eq ""){ + $rootDN = "cn=directory manager"; +} + +# +# get the logs +# +while($arg_count <= $#ARGV){ + $files[$file_count] = $ARGV[$arg_count]; + $file_count++; + $arg_count++; +} + +if($file_count == 0){ + if($reportStats){ + print "Usage error for option $reportStats, either the output file or access log is missing!\n\n"; + } else { + print "There are no access logs specified!\n\n"; + } + exit 1; +} + +if ($sizeCount eq "all"){$sizeCount = "100000";} + +####################################### +# # +# Initialize Arrays and variables # +# # +####################################### + +print "\nAccess Log Analyzer $logversion\n"; +print "\nCommand: logconv.pl @ARGV\n\n"; + +$rootDNBindCount = 0; +$anonymousBindCount = 0; +$unindexedSrchCount = 0; +$vlvNotesCount= 0; +$srchCount = 0; +$fdTaken = 0; +$fdReturned = 0; +$highestFdTaken = 0; +$unbindCount = 0; +$cmpCount = 0; +$modCount = 0; +$delCount = 0; +$addCount = 0; +$modrdnCount = 0; +$abandonCount = 0; +$extopCount = 0; +$vlvCount = 0; +$errorCount = 0; +$proxiedAuthCount = 0; +$serverRestartCount = 0; +$resourceUnavailCount = 0; +$brokenPipeCount = 0; +$v2BindCount = 0; +$v3BindCount = 0; +$vlvSortCount = 0; +$connResetByPeerCount = 0; +$isVlvNotes = 0; +$successCount = 0; +$sslCount = 0; +$sslClientBindCount = 0; +$sslClientFailedCount = 0; +$objectclassTopCount= 0; +$pagedSearchCount = 0; +$bindCount = 0; +$filterCount = 0; +$baseCount = 0; +$scopeCount = 0; +$allOps = 0; +$allResults = 0; +$badPwdCount = 0; +$saslBindCount = 0; +$internalOpCount = 0; +$entryOpCount = 0; +$referralCount = 0; +$anyAttrs = 0; +$persistentSrchCount = 0; +$maxBerSizeCount = 0; +$connectionCount = 0; +$timerange = 0; +$simConnection = 0; +$maxsimConnection = 0; +$firstFile = 1; +$elapsedDays = 0; +$logCount = 0; +$limit = 25000; # number of lines processed to trigger output + +# hash files +$ATTR = "$dataLocation/attr.logconv"; +$RC = "$dataLocation/rc.logconv"; +$SRC = "$dataLocation/src.logconv"; +$RSRC = "$dataLocation/rsrc.logconv"; +$EXCOUNT = "$dataLocation/excount.logconv"; +$CONN_HASH = "$dataLocation/conn_hash.logconv"; +$IP_HASH = "$dataLocation/ip_hash.logconv"; +$CONNCOUNT = "$dataLocation/conncount.logconv"; +$NENTRIES = "$dataLocation/nentries.logconv"; +$FILTER = "$dataLocation/filter.logconv"; +$BASE = "$dataLocation/base.logconv"; +$DS6XBADPWD = "$dataLocation/ds6xbadpwd.logconv"; +$SASLMECH = "$dataLocation/saslmech.logconv"; +$BINDLIST = "$dataLocation/bindlist.logconv"; +$ETIME = "$dataLocation/etime.logconv"; +$OID = "$dataLocation/oid.logconv"; + +# array files +$SRCH_CONN = "$dataLocation/srchconn.logconv"; +$SRCH_OP = "$dataLocation/srchop.logconv"; +$DEL_CONN = "$dataLocation/delconn.logconv"; +$DEL_OP = "$dataLocation/delop.logconv"; +$MOD_CONN = "$dataLocation/modconn.logconv"; +$MOD_OP = "$dataLocation/modop.logconv"; +$ADD_CONN = "$dataLocation/addconn.logconv"; +$ADD_OP = "$dataLocation/addop.logconv"; +$MODRDN_CONN = "$dataLocation/modrdnconn.logconv"; +$MODRDN_OP = "$dataLocation/modrdnop.logconv"; +$CMP_CONN = "$dataLocation/cmpconn.logconv"; +$CMP_OP = "$dataLocation/cmpop.logconv"; +$TARGET_CONN = "$dataLocation/targetconn.logconv"; +$TARGET_OP = "$dataLocation/targetop.logconv"; +$MSGID = "$dataLocation/msgid.logconv"; +$BIND_CONN = "$dataLocation/bindconn.logconv"; +$BIND_OP = "$dataLocation/bindop.logconv"; +$UNBIND_CONN = "$dataLocation/unbindconn.logconv"; +$UNBIND_OP = "$dataLocation/unbindop.logconv"; +$EXT_CONN = "$dataLocation/extconn.logconv"; +$EXT_OP = "$dataLocation/extop.logconv"; +$NOTES_ETIME = "$dataLocation/notesetime.logconv"; +$NOTES_CONN = "$dataLocation/notesconn.logconv"; +$NOTES_OP = "$dataLocation/notesop.logconv"; +$NOTES_TIME = "$dataLocation/notestime.logconv"; +$NOTES_NENTRIES = "$dataLocation/notesnentries.logconv"; +$BADPWDCONN = "$dataLocation/badpwdconn.logconv"; +$BADPWDOP = "$dataLocation/badpwdop.logconv"; +$BADPWDIP = "$dataLocation/badpwdip.logconv"; + +# info files +$BINDINFO = "$dataLocation/bindinfo.logconv"; +$BASEINFO = "$dataLocation/baseinfo.logconv"; +$FILTERINFO = "$dataLocation/filterinfo.logconv"; +$SCOPEINFO = "$dataLocation/scopeinfo.logconv"; + +$err[0] = "Successful Operations\n"; +$err[1] = "Operations Error(s)\n"; +$err[2] = "Protocal Errors\n"; +$err[3] = "Time Limit Exceeded\n"; +$err[4] = "Size Limit Exceeded\n"; +$err[5] = "Compare False\n"; +$err[6] = "Compare True\n"; +$err[7] = "Strong Authentication Not Supported\n"; +$err[8] = "Strong Authentication Required\n"; +$err[9] = "Partial Results\n"; +$err[10] = "Referral Received\n"; +$err[11] = "Administrative Limit Exceeded (Look Through Limit)\n"; +$err[12] = "Unavailable Critical Extension\n"; +$err[13] = "Confidentiality Required\n"; +$err[14] = "SASL Bind in Progress\n"; +$err[16] = "No Such Attribute\n"; +$err[17] = "Undefined Type\n"; +$err[18] = "Inappropriate Matching\n"; +$err[19] = "Constraint Violation\n"; +$err[20] = "Type or Value Exists\n"; +$err[21] = "Invalid Syntax\n"; +$err[32] = "No Such Object\n"; +$err[33] = "Alias Problem\n"; +$err[34] = "Invalid DN Syntax\n"; +$err[35] = "Is Leaf\n"; +$err[36] = "Alias Deref Problem\n"; +$err[48] = "Inappropriate Authentication (No password presented, etc)\n"; +$err[49] = "Invalid Credentials (Bad Password)\n"; +$err[50] = "Insufficent (write) Privledges\n"; +$err[51] = "Busy\n"; +$err[52] = "Unavailable\n"; +$err[53] = "Unwilling To Perform\n"; +$err[54] = "Loop Detected\n"; +$err[60] = "Sort Control Missing\n"; +$err[61] = "Index Range Error\n"; +$err[64] = "Naming Violation\n"; +$err[65] = "Objectclass Violation\n"; +$err[66] = "Not Allowed on Non Leaf\n"; +$err[67] = "Not Allowed on RDN\n"; +$err[68] = "Already Exists\n"; +$err[69] = "No Objectclass Mods\n"; +$err[70] = "Results Too Large\n"; +$err[71] = "Effect Multiple DSA's\n"; +$err[80] = "Other :-)\n"; +$err[81] = "Server Down\n"; +$err[82] = "Local Error\n"; +$err[83] = "Encoding Error\n"; +$err[84] = "Decoding Error\n"; +$err[85] = "Timeout\n"; +$err[86] = "Authentication Unknown\n"; +$err[87] = "Filter Error\n"; +$err[88] = "User Canceled\n"; +$err[89] = "Parameter Error\n"; +$err[90] = "No Memory\n"; +$err[91] = "Connect Error\n"; +$err[92] = "Not Supported\n"; +$err[93] = "Control Not Found\n"; +$err[94] = "No Results Returned\n"; +$err[95] = "More Results To Return\n"; +$err[96] = "Client Loop\n"; +$err[97] = "Referral Limit Exceeded\n"; + + +$conn{"A1"} = "A1"; +$conn{"B1"} = "B1"; +$conn{"B4"} = "B4"; +$conn{"T1"} = "T1"; +$conn{"T2"} = "T2"; +$conn{"B2"} = "B2"; +$conn{"B3"} = "B3"; +$conn{"R1"} = "R1"; +$conn{"P1"} = "P1"; +$conn{"P2"} = "P2"; +$conn{"U1"} = "U1"; + +$connmsg{"A1"} = "Client Aborted Connections"; +$connmsg{"B1"} = "Bad Ber Tag Encountered"; +$connmsg{"B4"} = "Server failed to flush data (response) back to Client"; +$connmsg{"T1"} = "Idle Timeout Exceeded"; +$connmsg{"T2"} = "IO Block Timeout Exceeded or NTSSL Timeout"; +$connmsg{"B2"} = "Ber Too Big"; +$connmsg{"B3"} = "Ber Peek"; +$connmsg{"R1"} = "Revents"; +$connmsg{"P1"} = "Plugin"; +$connmsg{"P2"} = "Poll"; +$connmsg{"U1"} = "Cleanly Closed Connections"; + +%monthname = ( + "Jan" => 0, + "Feb" => 1, + "Mar" => 2, + "Apr" => 3, + "May" => 4, + "Jun" => 5, + "Jul" => 6, + "Aug" => 7, + "Sep" => 8, + "Oct" => 9, + "Nov" => 10, + "Dec" => 11, + +); + +openDataFiles(); + +########################################## +# # +# Parse Access Logs # +# # +########################################## + +if ($files[$#files] =~ m/access.rotationinfo/) { $file_count--; } + +print "Processing $file_count Access Log(s)...\n\n"; + +#print "Filename\t\t\t Total Lines\n"; +#print "--------------------------------------------------\n"; + +if ($file_count > 1 && $files[0] =~ /\/access$/){ + $files[$file_count] = $files[0]; + $file_count++; + $skipFirstFile = 1; +} +$logCount = $file_count; + +for ($count=0; $count < $file_count; $count++){ + # we moved access to the end of the list, so if its the first file skip it + if($file_count > 1 && $count == 0 && $skipFirstFile == 1){ + next; + } + $logsize = `wc -l $files[$count]`; + $logsize =~ /([0-9]+)/; + $linesProcessed = 0; $lineBlockCount = 0; + $logCount--; + if($logCount < 10 ){ + # add a zero for formatting purposes + $logCountStr = "0" . $logCount; + } else { + $logCountStr = $logCount; + } + print sprintf "[%s] %-30s\tlines: %7s\n",$logCountStr, $files[$count], $1; + + open(LOG,"$files[$count]") or do { openFailed($!, $files[$count]) }; + $firstline = "yes"; + while(<LOG>){ + unless ($endFlag) { + if ($firstline eq "yes"){ + if (/^\[/) { + $logline = $_; + $firstline = "no"; + } + $linesProcessed++;$lineBlockCount++; + } elsif (/^\[/ && $firstline eq "no"){ + &parseLine(); + $logline = $_; + } else { + $logline = $logline . $_; + $logline =~ s/\n//; + } + } + } + &parseLine(); + close (LOG); + print_stats_block( $s_stats ); + print_stats_block( $m_stats ); + $totalLineCount = $totalLineCount + $linesProcessed; + if($linesProcessed => $limit){print sprintf " %10s Lines Processed\n\n",--$linesProcessed;} +} + +print "\n\nTotal Log Lines Analysed: " . ($totalLineCount - 1) . "\n"; + +$allOps = $srchCount + $modCount + $addCount + $cmpCount + $delCount + $modrdnCount + $bindCount + $extopCount + $abandonCount + $vlvCount; + +################################################################## +# # +# Calculate the total elapsed time of the processed access logs # +# # +################################################################## + +# if we are using startTime & endTime then we need to clean it up for our processing + +if($startTime){ + if ($start =~ / *([0-9a-z:\/]+)/i){$start=$1;} +} +if($endTime){ + if ($end =~ / *([0-9a-z:\/]+)/i){$end =$1;} +} + +# +# Get the start time in seconds +# + +$logStart = $start; + +if ($logStart =~ / *([0-9A-Z\/]+)/i ){ + $logDate = $1; + @dateComps = split /\//, $logDate; + + $timeMonth = 1 +$monthname{$dateComps[1]}; + $timeMonth = $timeMonth * 3600 *24 * 30; + $timeDay= $dateComps[0] * 3600 *24; + $timeYear = $dateComps[2] *365 * 3600 * 24; + $dateTotal = $timeMonth + $timeDay + $timeYear; +} + +if ($logStart =~ / *(:[0-9:]+)/i ){ + $logTime = $1; + @timeComps = split /:/, $logTime; + + $timeHour = $timeComps[1] * 3600; + $timeMinute = $timeComps[2] *60; + $timeSecond = $timeComps[3]; + $timeTotal = $timeHour + $timeMinute + $timeSecond; +} + +$startTotal = $timeTotal + $dateTotal; + +# +# Get the end time in seconds +# + +$logEnd = $end; + +if ($logEnd =~ / *([0-9A-Z\/]+)/i ){ + $logDate = $1; + @dateComps = split /\//, $logDate; + + $endDay = $dateComps[0] *3600 * 24; + $endMonth = 1 + $monthname{$dateComps[1]}; + $endMonth = $endMonth * 3600 * 24 * 30; + $endYear = $endTotal + $dateComps[2] *365 * 3600 * 24 ; + $dateTotal = $endDay + $endMonth + $endYear; +} + +if ($logEnd =~ / *(:[0-9:]+)/i ){ + $logTime = $1; + @timeComps = split /:/, $logTime; + + $endHour = $timeComps[1] * 3600; + $endMinute = $timeComps[2] *60; + $endSecond = $timeComps[3]; + $timeTotal = $endHour + $endMinute + $endSecond; +} + +$endTotal = $timeTotal + $dateTotal; + +# +# Tally the numbers +# +$totalTimeInSecs = $endTotal - $startTotal; +$remainingTimeInSecs = $totalTimeInSecs; + +# +# Calculate the elapsed time +# + +# days +while(($remainingTimeInSecs - 86400) > 0){ + $elapsedDays++; + $remainingTimeInSecs = $remainingTimeInSecs - 86400; + +} + +# hours +while(($remainingTimeInSecs - 3600) > 0){ + $elapsedHours++; + $remainingTimeInSecs = $remainingTimeInSecs - 3600; +} + +# minutes +while($remainingTimeInSecs - 60 > 0){ + $elapsedMinutes++; + $remainingTimeInSecs = $remainingTimeInSecs - 60; +} + +# seconds +$elapsedSeconds = $remainingTimeInSecs; + +# Initialize empty values +if($elapsedHours eq ""){ + $elapsedHours = "0"; +} +if($elapsedMinutes eq ""){ + $elapsedMinutes = "0"; +} +if($elapsedSeconds eq ""){ + $elapsedSeconds = "0"; +} + +&closeDataFiles(); + + +##################################### +# # +# Display Basic Results # +# # +##################################### + + +print "\n\n----------- Access Log Output ------------\n"; +print "\nStart of Logs: $start\n"; +print "End of Logs: $end\n\n"; + +if($elapsedDays eq "0"){ + print "Processed Log Time: $elapsedHours Hours, $elapsedMinutes Minutes, $elapsedSeconds Seconds\n\n"; +} else { + print "Processed Log Time: $elapsedDays Days, $elapsedHours Hours, $elapsedMinutes Minutes, $elapsedSeconds Seconds\n\n"; +} + +# +# Check here if we are producing any unqiue reports +# + +if($reportBinds eq "yes"){ + &displayBindReport(); +} + +# +# Continue with standard report +# + +print "Restarts: $serverRestartCount\n"; +print "Total Connections: $connectionCount\n"; +print "SSL Connections: $sslCount\n"; +print "Peak Concurrent Connections: $maxsimConnection\n"; +print "Total Operations: $allOps\n"; +print "Total Results: $allResults\n"; +if ($allOps ne "0"){ + print sprintf "Overall Performance: %.1f%\n\n" , ($perf = ($tmp = ($allResults / $allOps)*100) > 100 ? 100.0 : $tmp) ; + } +else { + print "Overall Performance: No Operations to evaluate\n\n"; +} + +$searchStat = sprintf "(%.2f/sec) (%.2f/min)\n",($srchCount / $totalTimeInSecs), $srchCount / ($totalTimeInSecs/60); +$modStat = sprintf "(%.2f/sec) (%.2f/min)\n",$modCount / $totalTimeInSecs, $modCount/($totalTimeInSecs/60); +$addStat = sprintf "(%.2f/sec) (%.2f/min)\n",$addCount/$totalTimeInSecs, $addCount/($totalTimeInSecs/60); +$deleteStat = sprintf "(%.2f/sec) (%.2f/min)\n",$delCount/$totalTimeInSecs, $delCount/($totalTimeInSecs/60); +$modrdnStat = sprintf "(%.2f/sec) (%.2f/min)\n",$modrdnCount/$totalTimeInSecs, $modrdnCount/($totalTimeInSecs/60); +$compareStat = sprintf "(%.2f/sec) (%.2f/min)\n",$cmpCount/$totalTimeInSecs, $cmpCount/($totalTimeInSecs/60); +$bindCountStat = sprintf "(%.2f/sec) (%.2f/min)\n",$bindCount/$totalTimeInSecs, $bindCount/($totalTimeInSecs/60); + +format STDOUT = +Searches: @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< + $srchCount, $searchStat +Modifications: @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< + $modCount, $modStat +Adds: @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< + $addCount, $addStat +Deletes: @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< + $delCount, $deleteStat +Mod RDNs: @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< + $modrdnCount, $modrdnStat +Compares: @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< + $cmpCount, $compareStat +Binds: @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< + $bindCount $bindCountStat +. +write STDOUT; + +print "\n"; +print "Proxied Auth Operations: $proxiedAuthCount\n"; +print "Persistent Searches: $persistentSrchCount\n"; +print "Internal Operations: $internalOpCount\n"; +print "Entry Operations: $entryOpCount\n"; +print "Extended Operations: $extopCount\n"; +print "Abandoned Requests: $abandonCount\n"; +print "Smart Referrals Received: $referralCount\n"; +print "\n"; +print "VLV Operations: $vlvCount\n"; +print "VLV Unindexed Searches: $vlvNotesCount\n"; +print "SORT Operations: $vlvSortCount\n"; +print "\n"; +print "Entire Search Base Queries: $objectclassTopCount\n"; +print "Paged Searches: $pagedSearchCount\n"; +print "Unindexed Searches: $unindexedSrchCount\n"; +if ($verb eq "yes" || $usage =~ /u/){ + if ($unindexedSrchCount > 0){ + %conn_hash = getHashFromFile($CONN_HASH); + @notesConn = getArrayFromFile($NOTES_CONN); + @notesOp = getArrayFromFile($NOTES_OP); + @notesEtime = getArrayFromFile($NOTES_ETIME); + @notesTime = getArrayFromFile($NOTES_TIME); + @notesNentries = getArrayFromFile($NOTES_NENTRIES); + getInfoArraysFromFile($BASEINFO); + @base_val = @fileArray1; + @base_conn = @fileArray2; + @base_op = @fileArray3; + getInfoArraysFromFile($SCOPEINFO); + @scope_val = @fileArray1; + @scope_conn = @fileArray2; + @scope_op = @fileArray3; + getInfoArraysFromFile($FILTERINFO); + @filter_val = @fileArray1; + @filter_conn = @fileArray2; + @filter_op = @fileArray3; + + $notesCount = "1"; + for ($n = 0; $n <= $#notesEtime; $n++){ + @alreadyseenDN = (); + if($conn_hash{$notesConn[$n]} eq ""){ + $unindexedIp = "?"; + } else { + $unindexedIp = $conn_hash{$notesConn[$n]}; + } + print "\n Unindexed Search #".$notesCount."\n"; $notesCount++; + print " - Date/Time: $notesTime[$n]\n"; + print " - Connection Number: $notesConn[$n]\n"; + print " - Operation Number: $notesOp[$n]\n"; + print " - Etime: $notesEtime[$n]\n"; + print " - Nentries: $notesNentries[$n]\n"; + print " - IP Address: $unindexedIp\n"; + + for ($nnn = 0; $nnn < $baseCount; $nnn++){ + if ($notesConn[$n] eq $base_conn[$nnn] && $notesOp[$n] eq $base_op[$nnn]){ + print " - Search Base: $base_val[$nnn]\n"; + last; + } + } + for ($nnn = 0; $nnn < $scopeCount; $nnn++){ + if ($notesConn[$n] eq $scope_conn[$nnn] && $notesOp[$n] eq $scope_op[$nnn]){ + print " - Search Scope: $scope_val[$nnn]\n"; + last; + } + } + for ($nnn = 0; $nnn < $filterCount; $nnn++){ + if ($notesConn[$n] eq $filter_conn[$nnn] && $notesOp[$n] eq $filter_op[$nnn]){ + print " - Search Filter: $filter_val[$nnn]\n"; + last; + } + } + } + undef %conn_hash; + undef @notesConn; + undef @notesOp; + undef @notesEtime; + undef @notesTime; + undef @notesNentries; + undef @notesIp; + undef @filter_val; + undef @filter_conn; + undef @filter_op; + undef @base_val; + undef @base_conn; + undef @base_op; + undef @scope_val; + undef @scope_conn; + undef @scope_op; + } +} # end of unindexed search report + +print "\n"; +print "FDs Taken: $fdTaken\n"; +print "FDs Returned: $fdReturned\n"; +print "Highest FD Taken: $highestFdTaken\n\n"; +print "Broken Pipes: $brokenPipeCount\n"; +if ($brokenPipeCount > 0){ + foreach $key (sort { $rc{$b} <=> $rc{$a} } keys %rc) { + if ($rc{$key} > 0){ + if ($conn{$key} eq ""){$conn{$key} = "**Unknown**";} + push @etext, sprintf " - %-4s (%2s) %-40s\n",$rc{$key},$conn{$key},$connmsg{$key +}; + } + } + print @etext; + print "\n"; +} + +print "Connections Reset By Peer: $connResetByPeerCount\n"; +if ($connResetByPeerCount > 0){ + foreach $key (sort { $src{$b} <=> $src{$a} } keys %src) { + if ($src{$key} > 0){ + if ($conn{$key} eq ""){$conn{$key} = "**Unknown**";} + push @retext, sprintf " - %-4s (%2s) %-40s\n",$src{$key},$conn{$key},$connmsg{$key +}; + } + } + print @retext; + print "\n"; +} + +print "Resource Unavailable: $resourceUnavailCount\n"; +if ($resourceUnavailCount > 0){ + foreach $key (sort { $rsrc{$b} <=> $rsrc{$a} } keys %rsrc) { + if ($rsrc{$key} > 0){ + if ($conn{$key} eq ""){$conn{$key} = "**Resource Issue**";} + push @rtext, sprintf " - %-4s (%2s) %-40s\n",$rsrc{$key},$conn{$key},$connmsg{$key}; + } + } + print @rtext; +} +print "Max BER Size Exceeded: $maxBerSizeCount\n"; +print "\n"; +print "Binds: $bindCount\n"; +print "Unbinds: $unbindCount\n"; +print "\n LDAP v2 Binds: $v2BindCount\n"; +print " LDAP v3 Binds: $v3BindCount\n"; +print " SSL Client Binds: $sslClientBindCount\n"; +print " Failed SSL Client Binds: $sslClientFailedCount\n"; +print " SASL Binds: $saslBindCount\n"; +if ($saslBindCount > 0){ + foreach $saslb ( sort {$saslmech{$b} <=> $saslmech{$a} } (keys %saslmech) ){ + printf " %-4s %-12s\n",$saslmech{$saslb}, $saslb; + } +} + +print "\n Directory Manager Binds: $rootDNBindCount\n"; +print " Anonymous Binds: $anonymousBindCount\n"; +$otherBindCount = $bindCount -($rootDNBindCount + $anonymousBindCount); +print " Other Binds: $otherBindCount\n\n"; + +########################################################################## +# Verbose Logging Section # +########################################################################## + +################################### +# # +# Display Connection Latency # +# # +################################### + +if ($verb eq "yes" || $usage =~ /y/){ + print "\n\n----- Connection Latency Details -----\n\n"; + print " (in seconds)\t\t<=1\t2\t3\t4-5\t6-10\t11-15\t>15\n"; + print " --------------------------------------------------------------------------\n"; + print " (# of connections)\t"; + for ($i=0; $i <=$#latency; $i++) { + print "$latency[$i]\t"; + } +} + +################################### +# # +# Display Open Connections # +# # +################################### + +if ($verb eq "yes" || $usage =~ /p/){ + if ($openConnection[0] ne ""){ + print "\n\n----- Current Open Connection IDs ----- \n\n"; + for ($i=0; $i <= $#openConnection ; $i++) { + if ($openConnection[$i]) { + print "Conn Number: $i (" . getIPfromConn($i) . ")\n"; + } + } + } +} + +################################### +# # +# Display Error Codes # +# # +################################### + +if ($usage =~ /e/i || $verb eq "yes"){ + print "\n\n----- Errors -----\n"; + + %er = sort( {$b <=> $a} %er); + for ($i = 0; $i<98; $i++){ + if ($err[$i] ne "" && $errorCode[$i] >0) { + push @errtext, sprintf "%-8s %12s %-25s","err=$i",$errorCode[$i],$err[$i]; + } + } + + for ($i = 0; $i < $#errtext; $i++){ + for ($ii = 0; $ii < $#errtext; $ii++){ + $yy="0"; + $zz="0"; + while ($errtext[$ii] =~ /(\w+)\s/g){ + $errornum[$yy]="$1"; + $yy++; + } + while ($errtext[$ii+1] =~ /(\w+)\s/g){ + $errornum2[$zz]="$1"; + $zz++; + } + if ($errornum2[1] > $errornum[1]){ + $tmp = $errtext[$ii]; + $errtext[$ii] = $errtext[$ii+1]; + $errtext[$ii+1] = $tmp; + } + } + } + for ($i = 0; $i <= $#errtext; $i++){ + $errtext[$i] =~ s/\n//g; + print "\n" . $errtext[$i]; + } +} + +#################################### +# # +# Print Failed Logins # +# # +#################################### + +if ($verb eq "yes" || $usage =~ /f/ ){ + if ($badPwdCount > 0){ + print "\n\n----- Top $sizeCount Failed Logins ------\n\n"; + + if ($ds6x eq "true"){ + %ds6xbadpwd = getCounterHashFromFile($DS6XBADPWD); + $ds6loop = 0; + foreach $ds6bp (sort { $ds6xbadpwd{$b} <=> $ds6xbadpwd{$a} } keys %ds6xbadpwd) { + if ($eloop > $sizeCount){ last; } + printf "%-4s %-40s\n", $ds6xbadpwd{$ds6bp}, $ds6bp; + $ds6loop++; + } + undef %ds6xbadpwd; + } else { + getInfoArraysFromFile($BINDINFO); + @bindVal = @fileArray1; + @bindConn = @fileArray2; + @bindOp = @fileArray3; + @badPasswordConn = getArrayFromFile($BADPWDCONN); + @badPasswordOp = getArrayFromFile($BADPWDOP); + @badPasswordIp = getArrayFromFile($BADPWDIP); + for ($ii =0 ; $ii < $badPwdCount; $ii++){ + for ($i = 0; $i < $bindCount; $i++){ + if ($badPasswordConn[$ii] eq $bindConn[$i] && $badPasswordOp[$ii] eq $bindOp[$i] ){ + $badPassword{ "$bindVal[$i]" } = $badPassword{ "$bindVal[$i]" } + 1; + } + } + } + # sort the new hash of $badPassword{} + $bpTotal = 0; + $bpCount = 0; + foreach $badpw (sort {$badPassword{$b} <=> $badPassword{$a} } keys %badPassword){ + if ($bpCount > $sizeCount){ last;} + $bpCount++; + $bpTotal = $bpTotal + $badPassword{"$badpw"}; + printf "%-4s %-40s\n", $badPassword{"$badpw"}, $badpw; + } + print "\nFrom the IP address(s) :\n\n"; + for ($i=0; $i<$badPwdCount; $i++) { + print "\t\t$badPasswordIp[$i]\n"; + } + if ($bpTotal > $badPwdCount){ + print "\n** Warning : Wrongly reported failed login attempts : ". ($bpTotal - $badPwdCount) . "\n"; + } + undef @bindVal; + undef @bindConn; + undef @bindOp; + undef @badPasswordConn; + undef @badPasswordOp; + undef @badPasswordIp; + } # this ends the if $ds6x = true + } +} + +#################################### +# # +# Print Connection Codes # +# # +#################################### + + +if ($connCodeCount > 0){ + if ($usage =~ /c/i || $verb eq "yes"){ + print "\n\n----- Total Connection Codes -----\n\n"; + %conncount = &getCounterHashFromFile($CONNCOUNT); + + foreach $key (sort { $conncount{$b} <=> $conncount{$a} } keys %conncount) { + if ($conncount{$key} > 0){ + push @conntext, sprintf "%-4s %6s %-40s\n",$key,$conncount{$key},$connmsg{ $key }; + } + } + print @conntext; + undef %conncount; + } +} + +######################################## +# # +# Gather and Process all unique IPs # +# # +######################################## + +if ($usage =~ /i/i || $verb eq "yes"){ + %ip_hash = getTwoDimHashFromFile($IP_HASH); + %exCount = getCounterHashFromFile($EXCOUNT); + @ipkeys = keys %ip_hash; + @exxCount = keys %exCount; + $ip_count = ($#ipkeys + 1)-($#exxCount + 1); + if ($ip_count > 0){ + print "\n\n----- Top $sizeCount Clients -----\n\n"; + print "Number of Clients: $ip_count\n\n"; + foreach $key (sort { $ip_hash{$b}{"count"} <=> $ip_hash{$a}{"count"} } keys %ip_hash) { + $exc = "no"; + if ($ccount > $sizeCount){ last;} + $ccount++; + for ($xxx =0; $xxx <= $#excludeIP; $xxx++){ + if ($excludeIP[$xxx] eq $key){$exc = "yes";} + } + if ($exc ne "yes"){ + if ($ip_hash{ $key }{"count"} eq ""){$ip_hash{ $key }{"count"} = "?";} + printf "[%s] Client: %s\n",$ccount, $key; + printf "%10s - Connections\n", $ip_hash{ $key }{"count"}; + foreach $code (sort { $ip_hash{ $key }{$b} <=> $ip_hash{ $key }{$a} } keys %{$ip_hash{ $key }}) { + if ($code eq 'count' ) { next; } + printf "%10s - %s (%s)\n", $ip_hash{ $key }{ $code }, $code, $connmsg{ $code }; + } + print "\n"; + } + } + } + undef %exCount; + undef %ip_hash; +} + +################################### +# # +# Gather All unique Bind DN's # +# # +################################### + +if ($usage =~ /b/i || $verb eq "yes"){ + %bindlist = getCounterHashFromFile($BINDLIST); + @bindkeys = keys %bindlist; + $bind_count = $#bindkeys + 1; + if ($bind_count > 0){ + print "\n\n----- Top $sizeCount Bind DN's -----\n\n"; + print "Number of Unique Bind DN's: $bind_count\n\n"; + $bindcount = 0; + foreach $dn (sort { $bindlist{$b} <=> $bindlist{$a} } keys %bindlist) { + if ($bindcount < $sizeCount){ + printf "%-8s %-40s\n", $bindlist{ $dn },$dn; + } + $bindcount++; + } + } + undef %bindlist; +} + +######################################### +# # +# Gather and process search bases # +# # +######################################### + +if ($usage =~ /a/i || $verb eq "yes"){ + %base = getCounterHashFromFile($BASE); + @basekeys = keys %base; + $base_count = $#basekeys + 1; + if ($base_count > 0){ + print "\n\n----- Top $sizeCount Search Bases -----\n\n"; + print "Number of Unique Search Bases: $base_count\n\n"; + $basecount = 0; + foreach $bas (sort { $base{$b} <=> $base{$a} } keys %base) { + if ($basecount < $sizeCount){ + printf "%-8s %-40s\n", $base{ $bas },$bas; + } + $basecount++; + } + } + undef %base; +} + +######################################### +# # +# Gather and process search filters # +# # +######################################### + +if ($usage =~ /l/ || $verb eq "yes"){ + %filter = getCounterHashFromFile($FILTER); + @filterkeys = keys %filter; + $filter_count = $#filterkeys + 1; + if ($filter_count > 0){ + print "\n\n----- Top $sizeCount Search Filters -----\n"; + print "\nNumber of Unique Search Filters: $filter_count\n\n"; + $filtercount = 0; + foreach $filt (sort { $filter{$b} <=> $filter{$a} } keys %filter){ + if ($filtercount < $sizeCount){ + printf "%-8s %-40s\n", $filter{$filt}, $filt; + } + $filtercount++; + } + } + undef %filter; +} + +######################################### +# # +# Gather and Process the unique etimes # +# # +######################################### + +if ($usage =~ /t/i || $verb eq "yes"){ + %etime = getCounterHashFromFile($ETIME); + # + # print most often etimes + # + print "\n\n----- Top $sizeCount Most Frequent etimes -----\n\n"; + $eloop = 0; + foreach $et (sort { $etime{$b} <=> $etime{$a} } keys %etime) { + if ($eloop == $sizeCount) { last; } + if ($retime ne "2"){ + $first = $et; + $retime = "2"; + } + printf "%-8s %-12s\n", $etime{ $et }, "etime=$et"; + $eloop++; + } + # + # print longest etimes + # + print "\n\n----- Top $sizeCount Longest etimes -----\n\n"; + $eloop = 0; + foreach $et (sort { $b <=> $a } (keys %etime)) { + if ($eloop == $sizeCount) { last; } + printf "%-12s %-10s\n","etime=$et",$etime{ $et }; + $eloop++; + } + undef %etime; +} + +####################################### +# # +# Gather and Process unique nentries # +# # +####################################### + + +if ($usage =~ /n/i || $verb eq "yes"){ + %nentries = getCounterHashFromFile($NENTRIES); + print "\n\n----- Top $sizeCount Largest nentries -----\n\n"; + $eloop = 0; + foreach $nentry (sort { $b <=> $a } (keys %nentries)){ + if ($eloop == $sizeCount) { last; } + printf "%-18s %12s\n","nentries=$nentry", $nentries{ $nentry }; + $eloop++; + } + print "\n\n----- Top $sizeCount Most returned nentries -----\n\n"; + $eloop = 0; + foreach $nentry (sort { $nentries{$b} <=> $nentries{$a} } (keys %nentries)){ + if ($eloop == $sizeCount) { last; } + printf "%-12s %-14s\n", $nentries{ $nentry }, "nentries=$nentry"; + $eloop++; + } + print "\n"; + undef %nentries; +} + +########################################## +# # +# Gather and process extended operations # +# # +########################################## + +if ($usage =~ /x/i || $verb eq "yes"){ + if ($extopCount > 0){ + %oid = getCounterHashFromFile($OID); + print "\n\n----- Extended Operations -----\n\n"; + foreach $oids (sort { $oid{$b} <=> $oid{$a} } (keys %oid) ){ + if ($oids eq "2.16.840.1.113730.3.5.1"){ $oidmessage = "Transaction Request"} #depreciated? + elsif ($oids eq "2.16.840.1.113730.3.5.2"){ $oidmessage = "Transaction Response"} #depreciated? + elsif ($oids eq "2.16.840.1.113730.3.5.3"){ $oidmessage = "Start Replication Request (incremental update)"} + elsif ($oids eq "2.16.840.1.113730.3.5.4"){ $oidmessage = "Replication Response"} + elsif ($oids eq "2.16.840.1.113730.3.5.5"){ $oidmessage = "End Replication Request (incremental update)"} + elsif ($oids eq "2.16.840.1.113730.3.5.6"){ $oidmessage = "Replication Entry Request"} + elsif ($oids eq "2.16.840.1.113730.3.5.7"){ $oidmessage = "Start Bulk Import"} + elsif ($oids eq "2.16.840.1.113730.3.5.8"){ $oidmessage = "Finished Bulk Import"} + elsif ($oids eq "2.16.840.1.113730.3.5.9"){ $oidmessage = "DS71 Replication Entry Request"} + elsif ($oids eq "2.16.840.1.113730.3.6.1"){ $oidmessage = "Incremental Update Replication Protocol"} + elsif ($oids eq "2.16.840.1.113730.3.6.2"){ $oidmessage = "Total Update Replication Protocol (Initialization)"} + elsif ($oids eq "2.16.840.1.113730.3.4.13"){ $oidmessage = "Replication Update Info Control"} + elsif ($oids eq "2.16.840.1.113730.3.6.4"){ $oidmessage = "DS71 Replication Incremental Update Protocol"} + elsif ($oids eq "2.16.840.1.113730.3.6.3"){ $oidmessage = "DS71 Replication Total Update Protocol"} + elsif ($oids eq "2.16.840.1.113730.3.5.12"){ $oidmessage = "DS90 Start Replication Request"} + elsif ($oids eq "2.16.840.1.113730.3.5.13"){ $oidmessage = "DS90 Replication Response"} + elsif ($oids eq "1.2.840.113556.1.4.841"){ $oidmessage = "Replication Dirsync Control"} + elsif ($oids eq "1.2.840.113556.1.4.417"){ $oidmessage = "Replication Return Deleted Objects"} + elsif ($oids eq "1.2.840.113556.1.4.1670"){ $oidmessage = "Replication WIN2K3 Active Directory"} + elsif ($oids eq "2.16.840.1.113730.3.6.5"){ $oidmessage = "Replication CleanAllRUV"} + elsif ($oids eq "2.16.840.1.113730.3.6.6"){ $oidmessage = "Replication Abort CleanAllRUV"} + elsif ($oids eq "2.16.840.1.113730.3.6.7"){ $oidmessage = "Replication CleanAllRUV Get MaxCSN"} + elsif ($oids eq "2.16.840.1.113730.3.6.8"){ $oidmessage = "Replication CleanAllRUV Check Status"} + elsif ($oids eq "2.16.840.1.113730.3.5.10"){ $oidmessage = "DNA Plugin Request"} + elsif ($oids eq "2.16.840.1.113730.3.5.11"){ $oidmessage = "DNA Plugin Response"} + elsif ($oids eq "1.3.6.1.4.1.1466.20037"){ $oidmessage = "Start TLS"} + elsif ($oids eq "1.3.6.1.4.1.4203.1.11.1"){ $oidmessage = "Password Modify"} + elsif ($oids eq "2.16.840.1.113730.3.4.20"){ $oidmessage = "MTN Control Use One Backend"} + else {$oidmessage = "Other"} + printf "%-6s %-23s %-60s\n", $oid{ $oids }, $oids, $oidmessage; + } + undef %oid; + } +} + +############################################ +# # +# Print most commonly requested attributes # +# # +############################################ + +if ($usage =~ /r/i || $verb eq "yes"){ + if ($anyAttrs > 0){ + %attr = getCounterHashFromFile($ATTR); + print "\n\n----- Top $sizeCount Most Requested Attributes -----\n\n"; + $eloop = 0; + foreach $mostAttr (sort { $attr{$b} <=> $attr{$a} } (keys %attr) ){ + if ($eloop eq $sizeCount){ last; } + printf "%-10s %-19s\n", $attr{$mostAttr}, $mostAttr; + $eloop++; + } + undef %attr; + } +} + +############################# +# # +# abandoned operation stats # +# # +############################# + +if ($usage =~ /g/i || $verb eq "yes"){ + $abandonTotal = $srchCount + $delCount + $modCount + $addCount + $modrdnCount + $bindCount + $extopCount + $cmpCount; + if ($verb eq "yes" && $abandonCount > 0 && $abandonTotal > 0){ + %conn_hash = getHashFromFile($CONN_HASH); + @srchConn = getArrayFromFile($SRCH_CONN); + @srchOp = getArrayFromFile($SRCH_OP); + @delConn = getArrayFromFile($DEL_CONN); + @delOp = getArrayFromFile($DEL_OP); + @targetConn = getArrayFromFile($TARGET_CONN); + @targetOp = getArrayFromFile($TARGET_OP); + @msgid = getArrayFromFile($MSGID); + @addConn = getArrayFromFile($ADD_CONN); + @addOp = getArrayFromFile($ADD_OP); + @modConn = getArrayFromFile($MOD_CONN); + @modOp = getArrayFromFile($MOD_OP); + @cmpConn = getArrayFromFile($CMP_CONN); + @cmpOp = getArrayFromFile($CMP_OP); + @modrdnConn = getArrayFromFile($MODRDN_CONN); + @modrdnOp = getArrayFromFile($MODRDN_OP); + @bindConn = getArrayFromFile($BIND_CONN); + @bindOp = getArrayFromFile($BIND_OP); + @unbindConn = getArrayFromFile($UNBIND_CONN); + @unbindOp = getArrayFromFile($UNBIND_OP); + @extConn = getArrayFromFile($EXT_CONN); + @extOp = getArrayFromFile($EXT_OP); + + print "\n\n----- Abandon Request Stats -----\n\n"; + + for ($g = 0; $g < $abandonCount; $g++){ + for ($sc = 0; $sc < $srchCount; $sc++){ + if ($srchConn[$sc] eq $targetConn[$g] && $srchOp[$sc] eq $targetOp[$g] ){ + print " - SRCH conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; + } + } + for ($dc = 0; $dc < $delCount; $dc++){ + if ($delConn[$dc] eq $targetConn[$g] && $delOp[$dc] eq $targetOp[$g]){ + print " - DEL conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; + } + } + for ($adc = 0; $adc < $addCount; $adc++){ + if ($addConn[$adc] eq $targetConn[$g] && $addOp[$adc] eq $targetOp[$g]){ + print " - ADD conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; + } + } + for ($mc = 0; $mc < $modCount; $mc++){ + if ($modConn[$mc] eq $targetConn[$g] && $modOp[$mc] eq $targetOp[$g]){ + print " - MOD conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; + } + } + for ($cc = 0; $cc < $cmpCount; $cc++){ + if ($cmpConn[$mdc] eq $targetConn[$g] && $cmpOp[$mdc] eq $targetOp[$g]){ + print " - CMP conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; + } + } + for ($mdc = 0; $mdc < $modrdnCount; $mdc++){ + if ($modrdnConn[$mdc] eq $targetConn[$g] && $modrdnOp[$mdc] eq $targetOp[$g]){ + print " - MODRDN conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; + } + } + for ($bcb = 0; $bcb < $bindCount; $bcb++){ + if ($bindConn[$bcb] eq $targetConn[$g] && $bindOp[$bcb] eq $targetOp[$g]){ + print " - BIND conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; + } + } + for ($ubc = 0; $ubc < $unbindCount; $ubc++){ + if ($unbindConn[$ubc] eq $targetConn[$g] && $unbindOp[$ubc] eq $targetOp[$g]){ + print " - UNBIND conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; + } + } + for ($ec = 0; $ec < $extopCount; $ec++){ + if ($extConn[$ec] eq $targetConn[$g] && $extOp[$ec] eq $targetOp[$g]){ + print " - EXT conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; + } + } + } + } +} +print "\n"; + +####################################### +# # +# Recommendations # +# # +####################################### + +if ($usage =~ /j/i || $verb eq "yes"){ + %conncount = getCounterHashFromFile($CONNCOUNT); + print "\n----- Recommendations -----\n"; + $recCount = "1"; + if ($unindexedSrchCount > 0){ + print "\n $recCount. You have unindexed searches, this can be caused from a search on an unindexed attribute, or your returned results exceeded the allidsthreshold. Unindexed searches are not recommended. To refuse unindexed searches, switch \'nsslapd-require-index\' to \'on\' under your database entry (e.g. cn=UserRoot,cn=ldbm database,cn=plugins,cn=config).\n"; + $recCount++; + } + if ($conncount{"T1"} > 0){ + print "\n $recCount. You have some connections that are are being closed by the idletimeout setting. You may want to increase the idletimeout if it is set low.\n"; + $recCount++; + } + if ($conncount{"T2"} > 0){ + print "\n $recCount. You have some coonections that are being closed by the ioblocktimeout setting. You may want to increase the ioblocktimeout.\n"; + $recCount++; + } + # compare binds to unbinds, if the difference is more than 30% of the binds, then report a issue + if (($bindCount - $unbindCount) > ($bindCount*.3)){ + print "\n $recCount. You have a significant difference between binds and unbinds. You may want to investigate this difference.\n"; + $recCount++; + } + # compare fds taken and return, if the difference is more than 30% report a issue + if (($fdTaken -$fdReturned) > ($fdTaken*.3)){ + print "\n $recCount. You have a significant difference between file descriptors taken and file descriptors returned. You may want to investigate this difference.\n"; + $recCount++; + } + if ($rootDNBindCount > ($bindCount *.2)){ + print "\n $recCount. You have a high number of Directory Manager binds. The Directory Manager account should only be used under certain circumstances. Avoid using this account for client applications.\n"; + $recCount++; + } + if ($errorCount > $successCount){ + print "\n $recCount. You have more unsuccessful operations than successful operations. You should investigate this difference.\n"; + $recCount++; + } + if ($conncount{"U1"} < ($connCodeCount - $conncount{"U1"})){ + print "\n $recCount. You have more abnormal connection codes than cleanly closed connections. You may want to investigate this difference.\n"; + $recCount++; + } + if ($first > 0){ + print "\n $recCount. You have a majority of etimes that are greater than zero, you may want to investigate this performance problem.\n"; + $recCount++; + } + if ($objectclassTopCount > ($srchCount *.25)){ + print "\n $recCount. You have a high number of searches that query the entire search base. Although this is not necessarily bad, it could be resource intensive if the search base contains many entries.\n"; + $recCount++; + } + if ($recCount == 1){ + print "\nNone.\n"; + } + print "\n"; + undef %conncount; +} + +# +# We're done, clean up the data files +# +removeDataFiles(); + +exit (0); + +####################### +# # +# Display Usage # +# # +####################### + +sub displayUsage { + + print "Usage:\n\n"; + + print " ./logconv.pl [-h] [-d|--rootdn <rootDN>] [-s|--sizeLimit <size limit>] [-v|verison] [-Vi|verbose]\n"; + print " [-S|--startTime <start time>] [-E|--endTime <end time>] \n"; + print " [-efcibaltnxrgjuyp] [ access log ... ... ]\n\n"; + + print "- Commandline Switches:\n\n"; + + print " -h, --help help/usage\n"; + print " -d, --rootDN <Directory Managers DN> default is \"cn=directory manager\"\n"; + print " -D, --data <Location for temporary data files> default is \"/tmp\"\n"; + print " -s, --sizeLimit <Number of results to return per catagory> default is 20\n"; + print " -X, --excludeIP <IP address to exclude from connection stats> E.g. Load balancers\n"; + print " -v, --version show version of tool\n"; + print " -S, --startTime <time to begin analyzing logfile from>\n"; + print " E.g. \"[28/Mar/2002:13:14:22 -0800]\"\n"; + print " -E, --endTime <time to stop analyzing logfile>\n"; + print " E.g. \"[28/Mar/2002:13:24:62 -0800]\"\n"; + print " -m, --reportFileSecs <CSV output file - per second stats>\n"; + print " -M, --reportFileMins <CSV output file - per minute stats>\n"; + print " -B, --bind <ALL | ANONYMOUS | \"Actual Bind DN\">\n"; + print " -V, --verbose <enable verbose output - includes all stats listed below>\n"; + print " -[efcibaltnxrgjuyp]\n\n"; + + print " e Error Code stats\n"; + print " f Failed Login Stats\n"; + print " c Connection Code Stats\n"; + print " i Client Stats\n"; + print " b Bind Stats\n"; + print " a Search Base Stats\n"; + print " l Search Filter Stats\n"; + print " t Etime Stats\n"; + print " n Nentries Stats\n"; + print " x Extended Operations\n"; + print " r Most Requested Attribute Stats\n"; + print " g Abandoned Operation Stats\n"; + print " j Recommendations\n"; + print " u Unindexed Search Stats\n"; + print " y Connection Latency Stats\n"; + print " p Open Connection ID Stats\n\n"; + + print " Examples:\n\n"; + + print " ./logconv.pl -s 10 -V /logs/access*\n\n"; + print " ./logconv.pl --rootDN cn=dm /logs/access*\n\n"; + print " ./logconv.pl --sizeLimit 50 -ibgju /logs/access*\n\n"; + print " ./logconv.pl -S \"\[28/Mar/2002:13:14:22 -0800\]\" --endTime \"\[28/Mar/2002:13:50:05 -0800\]\" -e /logs/access*\n\n"; + print " ./logconv.pl -m log-minute-stats-csv.out /logs/access*\n\n"; + print " ./logconv.pl -B ANONYMOUS /logs/access*\n\n"; + print " ./logconv.pl -B \"uid=mreynolds,dc=example,dc=com\" /logs/access*\n\n"; + + exit 1; +} + +###################################################### +# # +# Parsing Routines That Do The Actual Parsing Work # +# # +###################################################### + +sub +parseLine { + if($reportBinds eq "yes"){ + &parseLineBind(); + } else { + &parseLineNormal(); + } +} + +sub +parseLineBind { + $linesProcessed++; + $lineBlockCount++; + local $_ = $logline; + + if ($lineBlockCount >= $limit){ + print STDERR sprintf" %10s Lines Processed\n",$linesProcessed; + $lineBlockCount="0"; + } + + # skip blank lines + return if $_ =~ /^\s/; + + if($firstFile == 1 && $_ =~ /^\[/){ + $start = $_; + if ($start =~ / *([0-9a-z:\/]+)/i){$start=$1;} + $firstFile = 0; + } + if ($endFlag != 1 && $_ =~ /^\[/ && $_ =~ / *([0-9a-z:\/]+)/i){ + $end =$1; + } + if ($startTime && !$startFlag) { + if (index($_, $startTime) == 0) { + $startFlag = 1; + ($start) = $startTime =~ /\D*(\S*)/; + } else { + return; + } + } + if ($endTime && !$endFlag) { + if (index($_, $endTime) == 0) { + $endFlag = 1; + ($end) = $endTime =~ /\D*(\S*)/; + } + } + if ($_ =~ /connection from *([0-9A-Fa-f\.\:]+)/i ) { + for ($excl =0; $excl <= $#excludeIP; $excl++){ + if ($excludeIP[$excl] eq $1){ + $skip = "yes"; + last; + } + } + if ($skip eq "yes"){ + return ; + } + $ip = $1; + if ($_ =~ /conn= *([0-9]+)/i ){ + $connList{$ip} = $connList{$ip} . " $1 "; + } + return; + } + if (/ BIND/ && $_ =~ /dn=\"(.*)\" method/i ){ + if ($1 eq ""){ + $dn = "Anonymous"; + } else { + $dn = $1; + $dn =~ tr/A-Z/a-z/; + } + if($bindReportDN ne ""){ + if($dn ne $bindReportDN){ + # We are not looking for this DN, skip it + return; + } + } + $bindReport{$dn}{"binds"}++; + if ($bindReport{$dn}{"binds"} == 1){ + # For hashes we need to init the counters + $bindReport{$dn}{"srch"} = 0; + $bindReport{$dn}{"add"} = 0; + $bindReport{$dn}{"mod"} = 0; + $bindReport{$dn}{"del"} = 0; + $bindReport{$dn}{"cmp"} = 0; + $bindReport{$dn}{"ext"} = 0; + $bindReport{$dn}{"modrdn"} = 0; + $bindReport{$dn}{"failedBind"} = 0; + } + if ($_ =~ /conn= *([0-9]+)/i) { + $bindReport{$dn}{"conn"} = $bindReport{$dn}{"conn"} . " $1 "; + } + return; + } + if (/ RESULT err=49 /){ + processOpForBindReport("failedBind",$logline); + } + if (/ SRCH base=/){ + processOpForBindReport("srch",$logline); + } elsif (/ ADD dn=/){ + processOpForBindReport("add",$logline); + } elsif (/ MOD dn=/){ + processOpForBindReport("mod",$logline); + } elsif (/ DEL dn=/){ + processOpForBindReport("del",$logline); + } elsif (/ MODRDN dn=/){ + processOpForBindReport("modrdn",$logline); + } elsif (/ CMP dn=/){ + processOpForBindReport("cmp",$logline); + } elsif (/ EXT oid=/){ + processOpForBindReport("ext",$logline); + } +} + +sub +processOpForBindReport +{ + $op = @_[0]; + $data = @_[1]; + + if ($data =~ /conn= *([0-9]+)/i) { + foreach $dn (keys %bindReport){ + if ($bindReport{$dn}{"conn"} =~ / $1 /){ + $bindDN = $dn; + $bindReport{$bindDN}{$op}++; + return; + } + } + } +} + +sub parseLineNormal +{ + local $_ = $logline; + + # lines starting blank are restart + return if $_ =~ /^\s/; + + $linesProcessed++; + $lineBlockCount++; + if ($lineBlockCount >= $limit){ print STDERR sprintf" %10s Lines Processed\n",$linesProcessed; $lineBlockCount="0";} + + # gather/process the timestamp + if($firstFile == 1 && $_ =~ /^\[/){ + # if we are using startTime & endTime, this will get overwritten, which is ok + $start = $_; + if ($start =~ / *([0-9a-z:\/]+)/i){$start=$1;} + $firstFile = 0; + } + if ($endFlag != 1 && $_ =~ /^\[/ && $_ =~ / *([0-9a-z:\/]+)/i){$end =$1;} + if ($startTime && !$startFlag) { + if (index($_, $startTime) == 0) { + $startFlag = 1; + ($start) = $startTime =~ /\D*(\S*)/; + } else { + return; + } + } + if ($endTime && !$endFlag) { + if (index($_, $endTime) == 0) { + $endFlag = 1; + ($end) = $endTime =~ /\D*(\S*)/; + } + } + + # Additional performance stats + ($time, $tzone) = split (' ', $_); + if ($reportStat && $time ne $last_tm) + { + $last_tm = $time; + $time =~ s/\[//; + $tzone =~ s/\].*//; + + if($tzone ne $lastzone) + { + # tz offset change + $lastzone=$tzone; + ($sign,$hr,$min) = $tzone =~ m/(?)(\d\d)(\d\d)/; + $tzoff = $hr*3600 + $min*60; + $tzoff *= -1 + if $sign eq '-'; + # to be subtracted from converted values. + } + ($date, $hr, $min, $sec) = split (':', $time); + ($day, $mon, $yr) = split ('/', $date); + $newmin = timegm(0, $min, $hours, $day, $monthname{$mon}, $yr) - $tzoff; + $gmtime = $newmin + $sec; + print_stats_block( $s_stats ); + reset_stats_block( $s_stats, $gmtime, $time.' '.$tzone ); + if ($newmin != $last_min) + { + print_stats_block( $m_stats ); + $time =~ s/\d\d$/00/; + reset_stats_block( $m_stats, $newmin, $time.' '.$tzone ); + $last_min = $newmin; + } + } + + if (m/ RESULT err/){ + $allResults++; + if($reportStat){ inc_stats('results',$s_stats,$m_stats); } + } + if (m/ SRCH/){ + $srchCount++; + if($reportStat){ inc_stats('srch',$s_stats,$m_stats); } + if ($_ =~ / attrs=\"(.*)\"/i){ + $anyAttrs++; + $attrs = $1 . " "; + while ($attrs =~ /(\S+)\s/g){ + writeFile($ATTR, $1); + } + } + if (/ attrs=ALL/){ + writeFile($ATTR, "All Attributes"); + $anyAttrs++; + } + if ($verb eq "yes"){ + if ($_ =~ /conn= *([0-9]+)/i){ writeFile($SRCH_CONN, $1);} + if ($_ =~ /op= *([0-9]+)/i){ writeFile($SRCH_OP, $1);} + } + } + if (m/ DEL/){ + $delCount++; + if($reportStat){ inc_stats('del',$s_stats,$m_stats); } + if ($verb eq "yes"){ + if ($_ =~ /conn= *([0-9]+)/i){ writeFile($DEL_CONN, $1);} + if ($_ =~ /op= *([0-9]+)/i){ writeFile($DEL_OP, $1);} + } + } + if (m/ MOD dn=/){ + $modCount++; + if($reportStat){ inc_stats('mod',$s_stats,$m_stats); } + if ($verb eq "yes"){ + if ($_ =~ /conn= *([0-9]+)/i){ writeFile($MOD_CONN, $1);} + if ($_ =~ /op= *([0-9]+)/i){ writeFile($MOD_OP, $1); } + } + } + if (m/ ADD/){ + $addCount++; + if($reportStat){ inc_stats('add',$s_stats,$m_stats); } + if ($verb eq "yes"){ + if ($_ =~ /conn= *([0-9]+)/i){ writeFile($ADD_CONN, $1); } + if ($_ =~ /op= *([0-9]+)/i){ writeFile($ADD_OP, $1); } + } + } + if (m/ MODRDN/){ + $modrdnCount++; + if($reportStat){ inc_stats('modrdn',$s_stats,$m_stats); } + if ($verb eq "yes"){ + if ($_ =~ /conn= *([0-9]+)/i){ writeFile($MODRDN_CONN, $1); } + if ($_ =~ /op= *([0-9]+)/i){ writeFile($MODRDN_OP, $1); } + } + } + if (m/ CMP dn=/){ + $cmpCount++; + if($reportStat){ inc_stats('cmp',$s_stats,$m_stats); } + if ($verb eq "yes" || $usage =~ /g/i){ + if ($_ =~ /conn= *([0-9]+)/i){ writeFile($CMP_CONN, $1);} + if ($_ =~ /op= *([0-9]+)/i){ writeFile($CMP_OP, $1);} + } + } + if (m/ ABANDON /){ + $abandonCount++; + if($reportStat){ inc_stats('abandon',$s_stats,$m_stats); } + $allResults++; + if ($_ =~ /targetop= *([0-9a-zA-Z]+)/i ){ + writeFile($TARGET_OP, $1); + if ($_ =~ /conn= *([0-9]+)/i){ writeFile($TARGET_CONN, $1); } + if ($_ =~ /msgid= *([0-9]+)/i){ writeFile($MSGID, $1);} + } + } + if (m/ VLV /){ + if ($_ =~ /conn= *([0-9]+)/i){ $vlvconn[$vlvCount] = $1;} + if ($_ =~ /op= *([0-9]+)/i){ $vlvop[$vlvCount] = $1;} + $vlvCount++; + } + if (m/ authzid=/){ + $proxiedAuthCount++; + } + if (m/ SORT /){$vlvSortCount++} + if (m/ version=2/){$v2BindCount++} + if (m/ version=3/){$v3BindCount++} + if (m/ conn=1 fd=/){$serverRestartCount++} + if (m/ SSL connection from/){$sslCount++;} + if (m/ connection from/){ + $exc = "no"; + if ($_ =~ /connection from *([0-9A-Fa-f\.\:]+)/i ){ + for ($xxx =0; $xxx <= $#excludeIP; $xxx++){ + if ($excludeIP[$xxx] eq $1){$exc = "yes";} + } + if ($exc ne "yes"){ $connectionCount++;} + } + $simConnection++; + if ($simConnection > $maxsimConnection) { + $maxsimConnection = $simConnection; + } + ($connID) = $_ =~ /conn=(\d*)\s/; + $openConnection[$connID]++; + ($time, $tzone) = split (' ', $_); + ($date, $hr, $min, $sec) = split (':', $time); + ($day, $mon, $yr) = split ('/', $date); + $day =~ s/\[//; + $start_time_of_connection[$connID] = timegm($sec, $min, $hours, $day, $monthname{$mon}, $yr); + } + if (m/ SSL client bound as /){$sslClientBindCount++;} + if (m/ SSL failed to map client certificate to LDAP DN/){$sslClientFailedCount++;} + if (m/ fd=/ && m/slot=/){$fdTaken++} + if (m/ fd=/ && m/closed/){ + $fdReturned++; + $simConnection--; + + ($connID) = $_ =~ /conn=(\d*)\s/; + $openConnection[$connID]--; + $end_time_of_connection[$connID] = $gmtime; + $diff = $end_time_of_connection[$connID] - $start_time_of_connection[$connID]; + $start_time_of_connection[$connID] = $end_time_of_connection[$connID] = 0; + if ($diff <= 1) { $latency[0]++;} + if ($diff == 2) { $latency[1]++;} + if ($diff == 3) { $latency[2]++;} + if ($diff >= 4 && $diff <=5 ) { $latency[3]++;} + if ($diff >= 6 && $diff <=10 ) { $latency[4]++;} + if ($diff >= 11 && $diff <=15 ) { $latency[5]++;} + if ($diff >= 16) { $latency[6] ++;} + } + if (m/ BIND/ && $_ =~ /dn=\"(.*)\" method/i ){ + if($reportStat){ inc_stats('bind',$s_stats,$m_stats); } + $bindCount++; + if ($1 ne ""){ + $tmpp = $1; + $tmpp =~ tr/A-Z/a-z/; + writeFile($BINDLIST, $tmpp); + $bindVal = $tmpp; + if ($_ =~ /conn= *([0-9]+)/i) { $bindConn = $1; writeFile($BIND_CONN, $1);} + if ($_ =~ /op= *([0-9]+)/i) { $bindOp = $1; writeFile($BIND_OP, $1);} + if($usage =~ /f/ || $verb eq "yes"){ + # only need this for the failed bind report + writeFile($BINDINFO, "$bindVal ,, $bindConn ,, $bindOp"); + } + if($1 eq $rootDN){ + $rootDNBindCount++; + } + } else { + $anonymousBindCount++; + writeFile($BINDLIST, "Anonymous Binds"); + inc_stats('anonbind',$s_stats,$m_stats); + } + } + if (m/ UNBIND/){ + $unbindCount++; + if ($verb eq "yes"){ + if ($_ =~ /conn= *([0-9]+)/i){ writeFile($UNBIND_CONN, $1); } + if ($_ =~ /op= *([0-9]+)/i){ writeFile($UNBIND_OP, $1); } + } + } + if (m/ RESULT err=/ && m/ notes=P/){ + $pagedSearchCount++; + } + if (m/ notes=U/){ + if ($_ =~ /conn= *([0-9]+)/i){ + $con = $1; + if ($_ =~ /op= *([0-9]+)/i){ $op = $1;} + } + for ($i=0; $i <= $vlvCount;$i++){ + if ($vlvconn[$i] eq $con && $vlvop[$i] eq $op){ $vlvNotesCount++; $isVlvNotes="1";} + } + if($isVlvNotes == 0){ + # We don't want to record vlv unindexed searches for our regular "bad" + # unindexed search stat, as VLV unindexed searches aren't that bad + $unindexedSrchCount++; + if($reportStat){ inc_stats('notesu',$s_stats,$m_stats); } + } + if ($usage =~ /u/ || $verb eq "yes"){ + if ($isVlvNnotes == 0 ){ + if ($_ =~ /etime= *([0-9.]+)/i ){ + writeFile($NOTES_ETIME, $1); + } + if ($_ =~ /conn= *([0-9]+)/i){ + writeFile($NOTES_CONN, $1); + } + if ($_ =~ /op= *([0-9]+)/i){ + writeFile($NOTES_OP, $1); + } + if ($_ =~ / *([0-9a-z:\/]+)/i){ + writeFile($NOTES_TIME, $1); + } + if ($_ =~ /nentries= *([0-9]+)/i ){ + writeFile($NOTES_NENTRIES, $1); + } + } + } + $isVlvNotes = 0; + } + if (m/ closed error 32/){ + $brokenPipeCount++; + if (m/- T1/){ writeFile($RC,"T1"); } + elsif (m/- T2/){ writeFile($RC,"T2"); } + elsif (m/- A1/){ writeFile($RC,"A1"); } + elsif (m/- B1/){ writeFile($RC,"B1"); } + elsif (m/- B4/){ writeFile($RC,"B4"); } + elsif (m/- B2/){ writeFile($RC,"B2"); } + elsif (m/- B3/){ writeFile($RC,"B3"); } + elsif (m/- R1/){ writeFile($RC,"R1"); } + elsif (m/- P1/){ writeFile($RC,"P1"); } + elsif (m/- P1/){ writeFile($RC,"P2"); } + elsif (m/- U1/){ writeFile($RC,"U1"); } + else { writeFile($RC,"other"); } + } + if (m/ closed error 131/ || m/ closed error -5961/){ + $connResetByPeerCount++; + if (m/- T1/){ writeFile($SRC,"T1"); } + elsif (m/- T2/){ writeFile($SRC,"T2"); } + elsif (m/- A1/){ writeFile($SRC,"A1"); } + elsif (m/- B1/){ writeFile($SRC,"B1"); } + elsif (m/- B4/){ writeFile($SRC,"B4"); } + elsif (m/- B2/){ writeFile($SRC,"B2"); } + elsif (m/- B3/){ writeFile($SRC,"B3"); } + elsif (m/- R1/){ writeFile($SRC,"R1"); } + elsif (m/- P1/){ writeFile($SRC,"P1"); } + elsif (m/- P1/){ writeFile($SRC,"P2"); } + elsif (m/- U1/){ writeFile($SRC,"U1"); } + else { writeFile($SRC,"other"); } + } + if (m/ closed error 11/){ + $resourceUnavailCount++; + if (m/- T1/){ writeFile($RSRC,"T1"); } + elsif (m/- T2/){ writeFile($RSRC,"T2"); } + elsif (m/- A1/){ writeFile($RSRC,"A1"); } + elsif (m/- B1/){ writeFile($RSRC,"B1"); } + elsif (m/- B4/){ writeFile($RSRC,"B4"); } + elsif (m/- B2/){ writeFile($RSRC,"B2"); } + elsif (m/- B3/){ writeFile($RSRC,"B3"); } + elsif (m/- R1/){ writeFile($RSRC,"R1"); } + elsif (m/- P1/){ writeFile($RSRC,"P1"); } + elsif (m/- P1/){ writeFile($RSRC,"P2"); } + elsif (m/- U1/){ writeFile($RSRC,"U1"); } + else { writeFile($RSRC,"other"); } + } + if ($usage =~ /g/ || $usage =~ /c/ || $usage =~ /i/ || $verb eq "yes"){ + $exc = "no"; + if ($_ =~ /connection from *([0-9A-fa-f\.\:]+)/i ) { + for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + if ($1 eq $excludeIP[$xxx]){ + $exc = "yes"; + writeFile($EXCOUNT,$1); + } + } + $ip = $1; + writeFile($IP_HASH, "$ip count"); + if ($_ =~ /conn= *([0-9]+)/i ){ + if ($exc ne "yes"){ + writeFile($CONN_HASH, "$1 $ip"); + } + } + } + if (m/- A1/){ + if ($_ =~ /conn= *([0-9]+)/i) { + $exc = "no"; + $ip = getIPfromConn($1); + if ($ip eq ""){$ip = "Unknown_Host";} + for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + if ($ip eq $excludeIP[$xxx]){$exc = "yes";} + } + if ($exc ne "yes"){ + writeFile($IP_HASH, "$ip A1"); + writeFile($CONNCOUNT, "A1"); + $connCodeCount++; + } + } + } + if (m/- B1/){ + if ($_ =~ /conn= *([0-9]+)/i) { + $exc = "no"; + $ip = getIPfromConn($1); + if ($ip eq ""){$ip = "Unknown_Host";} + for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + if ($ip eq $excludeIP[$xxx]){$exc = "yes";} + } + if ($exc ne "yes"){ + writeFile($IP_HASH, "$ip B1"); + writeFile($CONNCOUNT, "B1"); + $connCodeCount++; + } + } + } + if (m/- B4/){ + if ($_ =~ /conn= *([0-9]+)/i) { + $exc = "no"; + $ip = getIPfromConn($1); + if ($ip eq ""){$ip = "Unknown_Host";} + for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + if ($ip eq $excludeIP[$xxx]){$exc = "yes";} + } + if ($exc ne "yes"){ + writeFile($IP_HASH, "$ip B4"); + writeFile($CONNCOUNT, "B4"); + $connCodeCount++; + } + } + } + if (m/- T1/){ + if ($_ =~ /conn= *([0-9]+)/i) { + $exc = "no"; + $ip = getIPfromConn($1); + if ($ip eq ""){$ip = "Unknown_Host";} + for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + if ($ip eq $excludeIP[$xxx]){$exc = "yes";} + } + if ($exc ne "yes"){ + writeFile($IP_HASH, "$ip T1"); + writeFile($CONNCOUNT, "T1"); + $connCodeCount++; + } + } + } + if (m/- T2/){ + if ($_ =~ /conn= *([0-9]+)/i) { + $exc = "no"; + $ip = getIPfromConn($1); + if ($ip eq ""){$ip = "Unknown_Host";} + for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + if ($ip eq $excludeIP[$xxx]){$exc = "yes";} + } + if ($exc ne "yes"){ + writeFile($IP_HASH, "$ip T2"); + writeFile($CONNCOUNT, "T2"); + $connCodeCount++; + } + } + } + if (m/- B2/){ + if ($_ =~ /conn= *([0-9]+)/i) { + $exc = "no"; + $ip = getIPfromConn($1); + $maxBerSizeCount++; + if ($ip eq ""){$ip = "Unknown_Host";} + for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + if ($ip eq $excludeIP[$xxx]){$exc = "yes";} + } + if ($exc ne "yes"){ + writeFile($IP_HASH, "$ip B2"); + writeFile($CONNCOUNT, "B2"); + $connCodeCount++; + } + } + } + if (m/- B3/){ + if ($_ =~ /conn= *([0-9]+)/i) { + $exc = "no"; + $ip = getIPfromConn($1); + if ($ip eq ""){$ip = "Unknown_Host";} + for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + if ($ip eq $excludeIP[$xxx]){$exc = "yes";} + } + if ($exc ne "yes"){ + writeFile($IP_HASH, "$ip B3"); + writeFile($CONNCOUNT, "B3"); + $connCodeCount++; + } + } + } + if (m/- R1/){ + if ($_ =~ /conn= *([0-9]+)/i) { + $exc = "no"; + $ip = getIPfromConn($1); + if ($ip eq ""){$ip = "Unknown_Host";} + for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + if ($ip eq $excludeIP[$xxx]){$exc = "yes";} + } + if ($exc ne "yes"){ + writeFile($IP_HASH, "$ip R1"); + writeFile($CONNCOUNT, "R1"); + $connCodeCount++; + } + } + } + if (m/- P1/){ + if ($_ =~ /conn= *([0-9]+)/i) { + $exc = "no"; + $ip = getIPfromConn($1); + if ($ip eq ""){$ip = "Unknown_Host";} + for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + if ($ip eq $excludeIP[$xxx]){$exc = "yes";} + } + if ($exc ne "yes"){ + writeFile($IP_HASH, "$ip P1"); + writeFile($CONNCOUNT, "P1"); + $connCodeCount++; + } + } + } + if (m/- P2/){ + if ($_ =~ /conn= *([0-9]+)/i) { + $exc = "no"; + $ip = getIPfromConn($1); + if ($ip eq ""){$ip = "Unknown_Host";} + for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + if ($ip eq $excludeIP[$xxx]){$exc = "yes";} + } + if ($exc ne "yes"){ + writeFile($IP_HASH, "$ip P2"); + writeFile($CONNCOUNT, "P2"); + $connCodeCount++; + } + } + } + if (m/- U1/){ + if ($_ =~ /conn= *([0-9]+)/i) { + $exc = "no"; + $ip = getIPfromConn($1); + if ($ip eq ""){$ip = "Unknown_Host";} + for ($xxx = 0; $xxx <= $#excludeIP; $xxx++){ + if ($ip eq $excludeIP[$xxx]){$exc = "yes";} + } + if ($exc ne "yes"){ + writeFile($IP_HASH, "$ip U1"); + writeFile($CONNCOUNT, "U1"); + $connCodeCount++; + } + } + } + } + if ($_ =~ /err= *([0-9]+)/i){ + $errorCode[$1]++; + if ($1 ne "0"){ $errorCount++;} + else { $successCount++;} + } + if ($_ =~ /etime= *([0-9.]+)/ ) { writeFile($ETIME, $1);} + if ($_ =~ / tag=101 / || $_ =~ / tag=111 / || $_ =~ / tag=100 / || $_ =~ / tag=115 /){ + if ($_ =~ / nentries= *([0-9]+)/i ){ writeFile($NENTRIES, $1); } + } + if (m/objectclass=\*/i || m/objectclass=top/i ){ + if (m/ scope=2 /){ $objectclassTopCount++;} + } + if (m/ EXT oid=/){ + $extopCount++; + if ($_ =~ /oid=\" *([0-9\.]+)/i ){ writeFile($OID,$1); } + if ($verb eq "yes"){ + if ($_ =~ /conn= *([0-9]+)/i){ writeFile($EXT_CONN, $1); } + if ($_ =~ /op= *([0-9]+)/i){ writeFile($EXT_OP, $1); } + } + } + if ($usage =~ /l/ || $verb eq "yes"){ + if (/ SRCH / && / attrs=/ && $_ =~ /filter=\"(.*)\" /i ){ + $tmpp = $1; + $tmpp =~ tr/A-Z/a-z/; + $tmpp =~ s/\\22/\"/g; + writeFile($FILTER, $tmpp); + $filterVal = $tmpp; + if ($_ =~ /conn= *([0-9]+)/i) { $filterConn = $1; } + if ($_ =~ /op= *([0-9]+)/i) { $filterOp = $1; } + } elsif (/ SRCH / && $_ =~ /filter=\"(.*)\"/i){ + $tmpp = $1; + $tmpp =~ tr/A-Z/a-z/; + $tmpp =~ s/\\22/\"/g; + writeFile($FILTER, $tmpp); + $filterVal = $tmpp; + if ($_ =~ /conn= *([0-9]+)/i) { $filterConn = $1; } + if ($_ =~ /op= *([0-9]+)/i) { $filterOp = $1; } + } + $filterCount++; + if($usage =~ /u/ || $verb eq "yes"){ + # we noly need this for the unindexed search report + writeFile($FILTERINFO, "$filterVal ,, $filterConn ,, $filterOp"); + } + } + if ($usage =~ /a/ || $verb eq "yes"){ + if (/ SRCH / && $_ =~ /base=\"(.*)\" scope/i ){ + if ($1 eq ""){ + $tmpp = "Root DSE"; + } else { + $tmpp = $1; + } + $tmpp =~ tr/A-Z/a-z/; + writeFile($BASE, $tmpp); + # + # grab the search bases & scope for potential unindexed searches + # + $baseVal = $tmpp; + if ($_ =~ /scope= *([0-9]+)/i) { + $scopeVal = $1; + } + if ($_ =~ /conn= *([0-9]+)/i) { + $baseConn = $1; + $scopeConn = $1; + } + if ($_ =~ /op= *([0-9]+)/i) { + $baseOp = $1; + $scopeOp = $1; + } + if($usage =~ /u/ || $verb eq "yes"){ + # we noly need this for the unindexed search report + writeFile($BASEINFO, "$baseVal ,, $baseConn ,, $baseOp"); + writeFile($SCOPEINFO, "$scopeVal ,, $scopeConn ,, $scopeOp"); + } + $baseCount++; + $scopeCount++; + } + } + if ($_ =~ /fd= *([0-9]+)/i ) { + $fds[$fdds] = $1; + if ($fds[$fdds] > $highestFdTaken) {$highestFdTaken = $fds[$fdds];} + $fdds++; + } + if ($usage =~ /f/ || $verb eq "yes"){ + if (/ err=49 tag=/ && / dn=\"/){ + if ($_ =~ /dn=\"(.*)\"/i ){ + writeFile($DS6XBADPWD, $1); + } + $ds6x = "true"; + $badPwdCount++; + } elsif (/ err=49 tag=/ ){ + if ($_ =~ /conn= *([0-9]+)/i ){ + writeFile($BADPWDCONN, $1); + $ip = getIPfromConn($1); + $badPwdCount++; + } + if ($_ =~ /op= *([0-9]+)/i ){ + writeFile($BADPWDOP, $1); + } + writeFile($BADPWDIP, $ip); + } + } + if (/ BIND / && /method=sasl/i){ + $saslBindCount++; + if ($_ =~ /mech=(.*)/i ){ + writeFile($SASLMECH, $1); + } + } + if (/ conn=Internal op=-1 / && !/ RESULT err=/){ $internalOpCount++; } + if (/ ENTRY dn=/ ){ $entryOpCount++; } + if (/ conn=/ && /op=/ && / REFERRAL/){ $referralCount++; } + if (/ options=persistent/){$persistentSrchCount++;} +} + +####################################### +# # +# CSV Helper Routines # +# # +####################################### +# # +# To convert the CSV to chart in OO # +# # +# * Select active rows and columns # +# * Insert -> Chart # +# * Chart type "XY (Scatter)" # +# * sub-type "Lines Only" # +# * select "Sort by X values" # +# * "Next" # +# * select "Data series in columns" # +# * select "First row as label" # +# * select "First column as label" # +# * "Next" # +# * "Next" # +# * "Finish" # +# # +####################################### + +sub +reset_stats_block +{ + my $stats = shift; + + $stats->{'last'} = shift || 0; + $stats->{'last_str'} = shift || ''; + + $stats->{'results'}=0; + $stats->{'srch'}=0; + $stats->{'add'}=0; + $stats->{'mod'}=0; + $stats->{'modrdn'}=0; + $stats->{'moddn'}=0; + $stats->{'cmp'}=0; + $stats->{'del'}=0; + $stats->{'abandon'}=0; + $stats->{'conns'}=0; + $stats->{'sslconns'}=0; + $stats->{'bind'}=0; + $stats->{'anonbind'}=0; + $stats->{'unbind'}=0; + $stats->{'notesu'}=0; + return; +} + +sub +new_stats_block +{ + my $name = shift || ''; + my $stats = { + 'active' => 0, + }; + if ($name){ + $stats->{'filename'} = $name; + $stats->{'fh'} = new IO::File; + $stats->{'active'} = open($stats->{'fh'},">$name"); + } + reset_stats_block( $stats ); + + return $stats; +} + +sub +print_stats_block +{ + foreach my $stats( @_ ){ + if ($stats->{'active'}){ + if ($stats->{'last'}){ + $stats->{'fh'}->print( + join(',', + $stats->{'last_str'}, + $stats->{'last'}, + $stats->{'results'}, + $stats->{'srch'}, + $stats->{'add'}, + $stats->{'mod'}, + $stats->{'modrdn'}, + $stats->{'moddn'}, + $stats->{'cmp'}, + $stats->{'del'}, + $stats->{'abandon'}, + $stats->{'conns'}, + $stats->{'sslconns'}, + $stats->{'bind'}, + $stats->{'anonbind'}, + $stats->{'unbind'}, + $stats->{'notesu'} ), + "\n" ); + } else { + $stats->{'fh'}->print( + "Time,time_t,Results,Search,Add,Mod,Modrdn,Delete,Abandon,". + "Connections,SSL Conns,Bind,Anon Bind,Unbind,Unindexed\n" + ); + } + } + } + return; +} + +sub +inc_stats +{ + my $n = shift; + foreach(@_){ + $_->{$n}++ + if exists $_->{$n}; + } + return; +} + +sub +displayBindReport +{ + # + # Loop for each DN - sort alphabetically + # + # Display all the IP addresses, then counts of all the operations it did + # + + print "\nBind Report\n"; + print "====================================================================\n\n"; + foreach $bindDN (sort { $bindReport{$a} <=> $bindReport{$b} } keys %bindReport) { + print("Bind DN: $bindDN\n"); + print("--------------------------------------------------------------------\n"); + print(" Client Addresses:\n\n"); + &printClients($bindReport{$bindDN}{"conn"}); + print("\n Operations Performed:\n\n"); + &printOpStats($bindDN); + print("\n"); + } + print "Done.\n"; + exit (0); +} + +sub +printClients +{ + @bindConns = &cleanConns(split(' ', @_[0])); + $IPcount = "1"; + + foreach $ip ( keys %connList ){ # Loop over all the IP addresses + foreach $bc (@bindConns){ # Loop over each bind conn number and compare it + if($connList{$ip} =~ / $bc /){ + print(" [$IPcount] $ip\n"); + $IPcount++; + last; + } + } + } +} + +sub +cleanConns +{ + @dirtyConns = @_; + $#cleanConns = -1; + $c = 0; + + for ($i = 0; $i <=$#dirtyConns; $i++){ + if($dirtyConns[$i] ne ""){ + $cleanConns[$c++] = $dirtyConns[$i]; + } + } + return @cleanConns; +} + +sub +printOpStats +{ + $dn = @_[0]; + + if( $bindReport{$dn}{"failedBind"} == 0 ){ + print(" Binds: " . $bindReport{$dn}{"binds"} . "\n"); + } else { + print(" Binds: " . $bindReport{$dn}{"binds"} . " (Invalid Credentials: " . $bindReport{$dn}{"failedBind"} . ")\n"); + } + print(" Searches: " . $bindReport{$dn}{"srch"} . "\n"); + print(" Modifies: " . $bindReport{$dn}{"mod"} . "\n"); + print(" Adds: " . $bindReport{$dn}{"add"} . "\n"); + print(" Deletes: " . $bindReport{$dn}{"del"} . "\n"); + print(" Compares: " . $bindReport{$dn}{"cmp"} . "\n"); + print(" ModRDNs: " . $bindReport{$dn}{"modrdn"} . "\n"); + print(" Ext Ops: " . $bindReport{$dn}{"ext"} . "\n\n"); +} + +####################### +# # +# Hash File Functions # +# # +####################### + +sub +openFailed +{ + $open_error = @_[0]; + $file_name = @_[1]; + closeDataFiles(); + removeDataFiles(); + die ("Can not open $file_name error ($open_error)"); +} + +sub +openDataFiles +{ + # hash files + open ($ATTR, ">$ATTR") or do { openFailed($!, $ATTR) }; + open ($RC, ">$RC") or do { openFailed($!, $RC) }; + open ($SRC, ">$SRC") or do { openFailed($!, $SRC) }; + open ($RSRC, ">$RSRC") or do { openFailed($!, $RSRC) }; + open ($EXCOUNT, ">$EXCOUNT") or do { openFailed($!, $EXCOUNT) }; + open ($CONN_HASH, ">$CONN_HASH") or do { openFailed($!, $CONN_HASH) }; + open ($IP_HASH, ">$IP_HASH") or do { openFailed($!, $IP_HASH) }; + open ($CONNCOUNT, ">$CONNCOUNT") or do { openFailed($!, $CONNCOUNT) }; + open ($NENTRIES, ">$NENTRIES") or do { openFailed($!, $NENTRIES) }; + open ($FILTER, ">$FILTER") or do { openFailed($!, $FILTER) }; + open ($BASE, ">$BASE") or do { openFailed($!, $BASE) }; + open ($DS6XBADPWD, ">$DS6XBADPWD") or do { openFailed($!, $DS6XBADPWD) }; + open ($SASLMECH, ">$SASLMECH") or do { openFailed($!, $SASLMECH) }; + open ($BINDLIST, ">$BINDLIST") or do { openFailed($!, $BINDLIST) }; + open ($ETIME, ">$ETIME") or do { openFailed($!, $ETIME) }; + open ($OID, ">$OID") or do { openFailed($!, $OID) }; + + # array files + open($SRCH_CONN,">$SRCH_CONN") or do { openFailed($!, $SRCH_CONN) }; + open($SRCH_OP, ">$SRCH_OP") or do { openFailed($!, $SRCH_OP) }; + open($DEL_CONN, ">$DEL_CONN") or do { openFailed($!, $DEL_CONN) }; + open($DEL_OP, ">$DEL_OP") or do { openFailed($!, $DEL_OP) }; + open($MOD_CONN, ">$MOD_CONN") or do { openFailed($!, $MOD_CONN) }; + open($MOD_OP, ">$MOD_OP") or do { openFailed($!, $MOD_OP) }; + open($ADD_CONN, ">$ADD_CONN") or do { openFailed($!, $ADD_CONN) }; + open($ADD_OP, ">$ADD_OP") or do { openFailed($!, $ADD_OP) }; + open($MODRDN_CONN, ">$MODRDN_CONN") or do { openFailed($!, $MODRDN_CONN) }; + open($MODRDN_OP, ">$MODRDN_OP") or do { openFailed($!, $MODRDN_OP) }; + open($CMP_CONN, ">$CMP_CONN") or do { openFailed($!, $CMP_CONN) }; + open($CMP_OP,">$CMP_OP") or do { openFailed($!, $CMP_OP) }; + open($TARGET_CONN, ">$TARGET_CONN") or do { openFailed($!, $TARGET_CONN) }; + open($TARGET_OP, ">$TARGET_OP") or do { openFailed($!, $TARGET_OP) }; + open($MSGID, ">$MSGID") or do { openFailed($!, $MSGID) }; + open($BIND_CONN, ">$BIND_CONN") or do { openFailed($!, $BIND_CONN) }; + open($BIND_OP, ">$BIND_OP") or do { openFailed($!, $BIND_OP) }; + open($UNBIND_CONN, ">$UNBIND_CONN") or do { openFailed($!, $UNBIND_CONN) }; + open($UNBIND_OP, ">$UNBIND_OP") or do { openFailed($!, $UNBIND_OP) }; + open($EXT_CONN, ">$EXT_CONN") or do { openFailed($!, $EXT_CONN) }; + open($EXT_OP, ">$EXT_OP") or do { openFailed($!, $EXT_OP) }; + open($NOTES_ETIME, ">$NOTES_ETIME") or do { openFailed($!, $NOTES_ETIME) }; + open($NOTES_CONN, ">$NOTES_CONN") or do { openFailed($!, $NOTES_CONN) }; + open($NOTES_OP, ">$NOTES_OP") or do { openFailed($!, $NOTES_OP) }; + open($NOTES_TIME, ">$NOTES_TIME") or do { openFailed($!, $NOTES_TIME) }; + open($NOTES_NENTRIES, ">$NOTES_NENTRIES") or do { openFailed($!, $NOTES_NENTRIES) }; + open($BADPWDCONN, ">$BADPWDCONN") or do { openFailed($!, $BADPWDCONN) }; + open($BADPWDOP, ">$BADPWDOP") or do { openFailed($!, $BADPWDOP) }; + open($BADPWDIP, ">$BADPWDIP") or do { openFailed($!, $NADPWDIP) }; + + # info files + open($BINDINFO, ">$BINDINFO") or do { openFailed($!, $BINDINFO) }; + open($BASEINFO, ">$BASEINFO") or do { openFailed($!, $BASEINFO) }; + open($SCOPEINFO, ">$SCOPEINFO") or do { openFailed($!, $SCOPEINFO) }; + open($FILTERINFO, ">$FILTERINFO") or do { openFailed($!, $FILTERINFO) }; +} + +sub +closeDataFiles +{ + close $ATTR; + close $RC; + close $SRC; + close $RSRC; + close $EXCOUNT; + close $CONN_HASH; + close $IP_HASH; + close $CONNCOUNT; + close $NENTRIES; + close $FILTER; + close $BASE; + close $DS6XBADPWD; + close $SASLMECH; + close $BINDLIST; + close $ETIME; + close $OID; + + # array files + close $SRCH_CONN; + close $SRCH_OP; + close $DEL_CONN; + close $DEL_OP; + close $MOD_CONN; + close $MOD_OP; + close $ADD_CONN; + close $ADD_OP; + close $MODRDN_CONN; + close $MODRDN_OP; + close $CMP_CONN; + close $CMP_OP; + close $TARGET_CONN; + close $TARGET_OP; + close $MSGID; + close $BIND_CONN; + close $BIND_OP; + close $UNBIND_CONN; + close $UNBIND_OP; + close $EXT_CONN; + close $EXT_OP; + close $NOTES_ETIME; + close $NOTES_CONN; + close $NOTES_OP; + close $NOTES_TIME; + close $NOTES_NENTRIES; + close $BADPWDCONN; + close $BADPWDOP; + close $BADPWDIP; + + # info files + close $BINDINFO; + close $BASEINFO; + close $SCOPEINFO; + close $FILTERINFO; +} + +sub +removeDataFiles +{ + unlink $ATTR; + unlink $RC; + unlink $SRC; + unlink $RSRC; + unlink $EXCOUNT; + unlink $CONN_HASH; + unlink $IP_HASH; + unlink $CONNCOUNT; + unlink $NENTRIES; + unlink $FILTER; + unlink $BASE; + unlink $DS6XBADPWD; + unlink $SASLMECH; + unlink $BINDLIST; + unlink $ETIME; + unlink $OID; + + # array files + unlink $SRCH_CONN; + unlink $SRCH_OP; + unlink $DEL_CONN; + unlink $DEL_OP; + unlink $MOD_CONN; + unlink $MOD_OP; + unlink $ADD_CONN; + unlink $ADD_OP; + unlink $MODRDN_CONN; + unlink $MODRDN_OP; + unlink $CMP_CONN; + unlink $CMP_OP; + unlink $TARGET_CONN; + unlink $TARGET_OP; + unlink $MSGID; + unlink $BIND_CONN; + unlink $BIND_OP; + unlink $UNBIND_CONN; + unlink $UNBIND_OP; + unlink $EXT_CONN; + unlink $EXT_OP; + unlink $NOTES_ETIME; + unlink $NOTES_CONN; + unlink $NOTES_OP; + unlink $NOTES_TIME; + unlink $NOTES_NENTRIES; + unlink $BADPWDCONN; + unlink $BADPWDOP; + unlink $BADPWDIP; + + # info files + unlink $BINDINFO; + unlink $BASEINFO; + unlink $SCOPEINFO; + unlink $FILTERINFO; +} + +sub +getIPfromConn +{ + $connip = @_[0]; + $retval = ""; + + close $CONN_HASH; # we can not read the file is its already open + open(CONN,"$CONN_HASH") or do { openFailed($!, $CONN_HASH) }; + while (<CONN>){ + if($_ =~ /$connip (.*)/){ + $retval = $1; + last; + } + } + close CONN; + #reopen file for writing(append) + open($CONN_HASH,">>$CONN_HASH") or do { openFailed($!, $CONN_HASH) }; + + return $retval; +} + +sub +writeFile +{ + $file = @_[0]; + $text = @_[1] . "\n"; + + print $file $text; +} + +# This hash file stores one value per line +sub +getCounterHashFromFile +{ + $file = @_[0]; + my %hash = (); + + open(FILE,"$file") or do { openFailed($!, $file) }; + while(<FILE>){ + chomp; + $hash{$_}++; + } + close FILE; + + return %hash; +} + +# this hash file stores two values per line (2 dimension hash) +sub +getTwoDimHashFromFile +{ + $file = @_[0]; + my %hash = (); + + open(FILE,"$file") or do { openFailed($!, $file) }; + while(<FILE>){ + @parts = split (' ', $_); + chomp(@parts); + $hash{$parts[0]}{$parts[1]}++; + } + close FILE; + + return %hash; +} + +# this hash file stores two values per line (1 dimension hash) +sub +getHashFromFile +{ + $file = @_[0]; + my %hash = (); + @parts = (); + + open(FILE,"$file") or do { openFailed($!, $file ) }; + while(<FILE>){ + @parts = split (' ',$_); + chomp(@parts); + $hash{$parts[0]} = $parts[1]; + } + close FILE; + + return %hash; +} + +# Return array of values from the file +sub +getArrayFromFile +{ + my @arry; + $file = @_[0]; + $array_count = 0; + + open(FILE,"$file") or do { openFailed($!, $file) }; + while(<FILE>){ + chomp; + $arry[$array_count] = $_; + $array_count++; + } + close FILE; + + return @arry; +} + +# build the three array +sub +getInfoArraysFromFile +{ + $file = @_[0]; + $array_count = 0; + @parts = (); + + open(FILE,"<$file") or do { openFailed($!, $file) }; + while(<FILE>){ + @parts = split (' ,, ',$_); + chomp(@parts); + if($#parts > 0){ + $fileArray1[$array_count] = $parts[0]; + $fileArray2[$array_count] = $parts[1]; + $fileArray3[$array_count] = $parts[2]; + $array_count++; + } + } + close FILE; +} + + + +####################################### +# # +# The End # +# # +####################################### + commit fc3c9192d37631948ca17fec3cd9bd2fed1ffdbc Author: Mark Reynolds <mreynolds(a)redhat.com&gt; Date: Tue Jan 15 12:41:23 2013 -0500 TIcket 419 - logconv.pl - improve memory management Remove old logconv.pl script - to make room for the new rewrite of logconv.pl https://fedorahosted.org/389/ticket/419 (cherry picked from commit 472373506eeca676dbbf9b9f56c2248761068752) diff --git a/ldap/admin/src/logconv.pl b/ldap/admin/src/logconv.pl deleted file mode 100755 index 0ec596a..0000000 --- a/ldap/admin/src/logconv.pl +++ /dev/null @@ -1,2271 +0,0 @@ -#!/usr/bin/env perl - -# -# BEGIN COPYRIGHT BLOCK -# This Program is free software; you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free Software -# Foundation; version 2 of the License. -# -# This Program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along with -# this Program; if not, write to the Free Software Foundation, Inc., 59 Temple -# Place, Suite 330, Boston, MA 02111-1307 USA. -# -# In addition, as a special exception, Red Hat, Inc. gives You the additional -# right to link the code of this Program with code not covered under the GNU -# General Public License ("Non-GPL Code") and to distribute linked combinations -# including the two, subject to the limitations in this paragraph. Non-GPL Code -# permitted under this exception must only link to the code of this Program -# through those well defined interfaces identified in the file named EXCEPTION -# found in the source code files (the "Approved Interfaces"). The files of -# Non-GPL Code may instantiate templates or use macros or inline functions from -# the Approved Interfaces without causing the resulting work to be covered by -# the GNU General Public License. Only Red Hat, Inc. may make changes or -# additions to the list of Approved Interfaces. You must obey the GNU General -# Public License in all respects for all of the Program code and other code used -# in conjunction with the Program except the Non-GPL Code covered by this -# exception. If you modify this file, you may extend this exception to your -# version of the file, but you are not obligated to do so. If you do not wish to -# provide this exception without modification, you must delete this exception -# statement from your version and license this file solely under the GPL without -# exception. -# -# -# Copyright (C) 2001 Sun Microsystems, Inc. Used by permission. -# Copyright (C) 2005 Red Hat, Inc. -# All rights reserved. -# END COPYRIGHT BLOCK -# - -# -# Check for usage -# -use Time::Local; -use IO::File; -use Getopt::Long; - -Getopt::Long::Configure ("bundling"); -Getopt::Long::Configure ("permute"); - -if ($#ARGV < 0){; -&displayUsage; -} - -####################################### -# # -# parse commandline switches # -# # -####################################### - -$x = "0"; -$fc = 0; -$sn = 0; -$logversion = "6.11"; -$sizeCount = "20"; -$startFlag = 0; -$startTime = 0; -$endFlag = 0; -$endTime = 0; -$s_stats = new_stats_block( ); -$m_stats = new_stats_block( ); -$report_opt = ""; - -GetOptions( - 'd|rootDN=s' => \$manager, - 'v|version' => sub { print "Access Log Analyzer v$logversion\n"; exit (0); }, - 'V|verbose' => sub { $verb = "yes"; }, - 'X|excludeIP=s' => \$exclude[$xi++], - 's|sizeLimit=s' => \$sizeCount, - 'S|startTime=s' => \$startTime, - 'E|endTime=s' => \$endTime, - 'B|bind=s' => sub { $reportBinds = "yes"; $bindReportDN=($_[1]) }, - 'm|reportFileSecs=s' => sub { my ($opt,$value) = @_; $s_stats = new_stats_block($value); $report_opt = "-m"; }, - 'M|reportFileMins=s' => sub { my ($opt,$value) = @_; $m_stats = new_stats_block($value); $report_opt = "-M"; }, - 'h|help' => sub { displayUsage() }, - # usage options '-efcibaltnxgjuiryp' - 'e' => sub { $usage = $usage . "e"; }, - 'f' => sub { $usage = $usage . "f"; }, - 'c' => sub { $usage = $usage . "c"; }, - 'i' => sub { $usage = $usage . "i"; }, - 'b' => sub { $usage = $usage . "b"; }, - 'a' => sub { $usage = $usage . "a"; }, - 'l' => sub { $usage = $usage . "l"; }, - 't' => sub { $usage = $usage . "t"; }, - 'n' => sub { $usage = $usage . "n"; }, - 'x' => sub { $usage = $usage . "x"; }, - 'g' => sub { $usage = $usage . "g"; }, - 'j' => sub { $usage = $usage . "j"; }, - 'u' => sub { $usage = $usage . "u"; }, - 'r' => sub { $usage = $usage . "r"; }, - 'y' => sub { $usage = $usage . "y"; }, - 'p' => sub { $usage = $usage . "p"; } -); - -# -# setup the report Bind DN if any -# -if($reportBinds eq "yes"){ - $bindReportDN =~ tr/A-Z/a-z/; - if($bindReportDN eq "all"){ - $bindReportDN = ""; - } - if($bindReportDN eq "anonymous"){ - $bindReportDN = "Anonymous"; - } -} - -# -# set the default root DN -# -if($manager eq ""){ - $manager = "cn=directory manager"; -} - -# -# get the logs -# -while($sn <= $#ARGV){ - $files[$fc] = $ARGV[$sn]; - $fc++; - $sn++; -} - -if($fc == 0){ - if($report_opt ne ""){ - print "Usage error for option $report_opt, either the output file or access log is missing!\n\n"; - } else { - print "There are no access logs specified!\n\n"; - } - exit 1; -} - -if ($sizeCount eq "all"){$sizeCount = "100000";} - -####################################### -# # -# Initialize Arrays and variables # -# # -####################################### - -print "\nAccess Log Analyzer $logversion\n"; -print "\nCommand : logconv.pl @ARGV\n\n"; - -$dirmgr = "0"; -$notes = "0"; -$vlvnotes= "0"; -$search = "0"; -$fdtake = "0"; -$fdreturn = "0"; -$highfd = "0"; -$bind = "0"; -$unbind = "0"; -$anony = "0"; -$mod = "0"; -$delete = "0"; -$add = "0"; -$modrdn = "0"; -$moddn = "0"; -$compare = "0"; -$proxiedAuth = "0"; -$restarts = "0"; -$resource = "0"; -$broken = "0"; -$vlv = "0"; -$version2 = "0"; -$version3 = "0"; -$sortvlv = "0"; -$reset = "0"; -$vet = "0"; -$v = "0"; -$errorck = "0"; -$errorsucc = "0"; -$sslconn = "0"; -$sslClientBind = "0"; -$sslClientFailed = "0"; -$objectclass= "0"; -$nc = "0"; -$no = "0"; -$nt = "0"; -$nb = "0"; -$bc = "0"; -$fcc = "0"; -$bcc = "0"; -$scc = "0"; -$nent = "0"; -$allOps = "0"; -$allResults = "0"; -$bpc = "0"; -$bpo = "0"; -$bpi = 0; -$abandon = "0"; -$mmasterop = "0"; -$extendedop = "0"; -$sasl = "0"; -$internal = "0"; -$entryOp = "0"; -$referral = "0"; -$anyAttrs = "0"; -$persistent = "0"; -$sconn = "0"; -$dconn = "0"; -$aconn = "0"; -$mconn = "0"; -$mdconn = "0"; -$mddconn = "0"; -$bconn = "0"; -$ubconn = "0"; -$econn = "0"; -$cconn = "0"; -$connectionCount = "0"; -$timerange = 0; -$simConnection = 0; -$maxsimConnection = 0; -$firstFile = "1"; -$elapsedDays = "0"; -$logCount = "0"; -$limit = "25000"; # number of lines processed to trigger output - -$err[0] = "Successful Operations\n"; -$err[1] = "Operations Error(s)\n"; -$err[2] = "Protocal Errors\n"; -$err[3] = "Time Limit Exceeded\n"; -$err[4] = "Size Limit Exceeded\n"; -$err[5] = "Compare False\n"; -$err[6] = "Compare True\n"; -$err[7] = "Strong Authentication Not Supported\n"; -$err[8] = "Strong Authentication Required\n"; -$err[9] = "Partial Results\n"; -$err[10] = "Referral Received\n"; -$err[11] = "Administrative Limit Exceeded (Look Through Limit)\n"; -$err[12] = "Unavailable Critical Extension\n"; -$err[13] = "Confidentiality Required\n"; -$err[14] = "SASL Bind in Progress\n"; -$err[16] = "No Such Attribute\n"; -$err[17] = "Undefined Type\n"; -$err[18] = "Inappropriate Matching\n"; -$err[19] = "Constraint Violation\n"; -$err[20] = "Type or Value Exists\n"; -$err[21] = "Invalid Syntax\n"; -$err[32] = "No Such Object\n"; -$err[33] = "Alias Problem\n"; -$err[34] = "Invalid DN Syntax\n"; -$err[35] = "Is Leaf\n"; -$err[36] = "Alias Deref Problem\n"; -$err[48] = "Inappropriate Authentication (No password presented, etc)\n"; -$err[49] = "Invalid Credentials (Bad Password)\n"; -$err[50] = "Insufficent (write) Privledges\n"; -$err[51] = "Busy\n"; -$err[52] = "Unavailable\n"; -$err[53] = "Unwilling To Perform\n"; -$err[54] = "Loop Detected\n"; -$err[60] = "Sort Control Missing\n"; -$err[61] = "Index Range Error\n"; -$err[64] = "Naming Violation\n"; -$err[65] = "Objectclass Violation\n"; -$err[66] = "Not Allowed on Non Leaf\n"; -$err[67] = "Not Allowed on RDN\n"; -$err[68] = "Already Exists\n"; -$err[69] = "No Objectclass Mods\n"; -$err[70] = "Results Too Large\n"; -$err[71] = "Effect Multiple DSA's\n"; -$err[80] = "Other\n"; -$err[81] = "Server Down\n"; -$err[82] = "Local Error\n"; -$err[83] = "Encoding Error\n"; -$err[84] = "Decoding Error\n"; -$err[85] = "Timeout\n"; -$err[86] = "Authentication Unknown\n"; -$err[87] = "Filter Error\n"; -$err[88] = "User Canceled\n"; -$err[89] = "Parameter Error\n"; -$err[90] = "No Memory\n"; -$err[91] = "Connect Error\n"; -$err[92] = "Not Supported\n"; -$err[93] = "Control Not Found\n"; -$err[94] = "No Results Returned\n"; -$err[95] = "More Results To Return\n"; -$err[96] = "Client Loop\n"; -$err[97] = "Referral Limit Exceeded\n"; - - -$conn{"A1"} = "A1"; -$conn{"B1"} = "B1"; -$conn{"B4"} = "B4"; -$conn{"T1"} = "T1"; -$conn{"T2"} = "T2"; -$conn{"B2"} = "B2"; -$conn{"B3"} = "B3"; -$conn{"R1"} = "R1"; -$conn{"P1"} = "P1"; -$conn{"P2"} = "P2"; -$conn{"U1"} = "U1"; - -$connmsg{"A1"} = "Client Aborted Connections"; -$connmsg{"B1"} = "Bad Ber Tag Encountered"; -$connmsg{"B4"} = "Server failed to flush data (response) back to Client"; -$connmsg{"T1"} = "Idle Timeout Exceeded"; -$connmsg{"T2"} = "IO Block Timeout Exceeded or NTSSL Timeout"; -$connmsg{"B2"} = "Ber Too Big"; -$connmsg{"B3"} = "Ber Peek"; -$connmsg{"R1"} = "Revents"; -$connmsg{"P1"} = "Plugin"; -$connmsg{"P2"} = "Poll"; -$connmsg{"U1"} = "Cleanly Closed Connections"; - -%monthname = ( - "Jan" => 0, - "Feb" => 1, - "Mar" => 2, - "Apr" => 3, - "May" => 4, - "Jun" => 5, - "Jul" => 6, - "Aug" => 7, - "Sep" => 8, - "Oct" => 9, - "Nov" => 10, - "Dec" => 11, - -); - -########################################## -# # -# Parse Access Logs # -# # -########################################## - -if ($files[$#files] =~ m/access.rotationinfo/) { $fc--; } - -print "Processing $fc Access Log(s)...\n\n"; - -print "Filename\t\t\t Total Lines\n"; -print "--------------------------------------------------\n"; - -$ofc = $fc; - -if ($fc > 1 && $files[0] =~ /\/access$/){ - $files[$fc] = $files[0]; - $fc++; - $skipFirstFile = "1"; -} - -for ($count=0; $count < $fc; $count++){ - # we moved access to the end of the list, so if its the first file skip it - if($fc > 1 && $count == 0 && $skipFirstFile eq "1"){ - next; - } - $logCount++; - $logsize = `wc -l $files[$count]`; - $logsize =~ /([0-9]+)/; - $ff="";$iff=""; - if($logCount < 10 ){ - # add a zero for formatting purposes - $lc = "0" . $logCount; - } else { - $lc = $logCount; - } - print sprintf "[%s] %-30s %7s\n",$lc, $files[$count], $1; - - open(LOG,"$files[$count]") || die "Error: Can't open file $infile: $!"; - - $firstline = "yes"; - while (<LOG>) { - unless ($endFlag) { - if ($firstline eq "yes"){ - if (/^\[/) { - $tline = $_; - $firstline = "no"; - } - $ff++;$iff++; - } elsif (/^\[/ && $firstline eq "no"){ - &parseLine($tline); - $tline = $_; - } else { - $tline = $tline . $_; - $tline =~ s/\n//; - } - } - } - &parseLine($tline); - close (LOG); - print_stats_block( $s_stats ); - print_stats_block( $m_stats ); - $tlc = $tlc + $ff; - if($ff => $limit){print sprintf " %10s Lines Processed\n\n",--$ff;} -} - -print "\n\nTotal Log Lines Analysed: " . ($tlc - 1) . "\n"; - -$allOps = $search + $mod + $add + $delete + $modrdn + $bind + $extendedop; - -################################################################## -# # -# Calculate the total elapsed time of the processed access logs # -# # -################################################################## - -# if we are using startTime & endTime then we need to clean it up for our processing - -if($startTime){ - if ($start =~ / *([0-9a-z:\/]+)/i){$start=$1;} -} -if($endTime){ - if ($end =~ / *([0-9a-z:\/]+)/i){$end =$1;} -} - -# -# Get the start time in seconds -# - -$logStart = $start; - -if ($logStart =~ / *([0-9A-Z\/]+)/i ){ - $logDate = $1; - @dateComps = split /\//, $logDate; - - $timeMonth = 1 +$monthname{$dateComps[1]}; - $timeMonth = $timeMonth * 3600 *24 * 30; - $timeDay= $dateComps[0] * 3600 *24; - $timeYear = $dateComps[2] *365 * 3600 * 24; - $dateTotal = $timeMonth + $timeDay + $timeYear; -} - -if ($logStart =~ / *(:[0-9:]+)/i ){ - $logTime = $1; - @timeComps = split /:/, $logTime; - - $timeHour = $timeComps[1] * 3600; - $timeMinute = $timeComps[2] *60; - $timeSecond = $timeComps[3]; - $timeTotal = $timeHour + $timeMinute + $timeSecond; -} - -$startTotal = $timeTotal + $dateTotal; - -# -# Get the end time in seconds -# - -$logEnd = $end; - -if ($logEnd =~ / *([0-9A-Z\/]+)/i ){ - $logDate = $1; - @dateComps = split /\//, $logDate; - - $endDay = $dateComps[0] *3600 * 24; - $endMonth = 1 + $monthname{$dateComps[1]}; - $endMonth = $endMonth * 3600 * 24 * 30; - $endYear = $endTotal + $dateComps[2] *365 * 3600 * 24 ; - $dateTotal = $endDay + $endMonth + $endYear; -} - -if ($logEnd =~ / *(:[0-9:]+)/i ){ - $logTime = $1; - @timeComps = split /:/, $logTime; - - $endHour = $timeComps[1] * 3600; - $endMinute = $timeComps[2] *60; - $endSecond = $timeComps[3]; - $timeTotal = $endHour + $endMinute + $endSecond; -} - -$endTotal = $timeTotal + $dateTotal; - -# -# Tally the numbers -# -$totalTimeInSecs = $endTotal - $startTotal; -$remainingTimeInSecs = $totalTimeInSecs; - -# -# Calculate the elapsed time -# - -# days -while(($remainingTimeInSecs - 86400) > 0){ - $elapsedDays++; - $remainingTimeInSecs = $remainingTimeInSecs - 86400; - -} - -# hours -while(($remainingTimeInSecs - 3600) > 0){ - $elapsedHours++; - $remainingTimeInSecs = $remainingTimeInSecs - 3600; -} - -# minutes -while($remainingTimeInSecs - 60 > 0){ - $elapsedMinutes++; - $remainingTimeInSecs = $remainingTimeInSecs - 60; -} - -#seconds -$elapsedSeconds = $remainingTimeInSecs; - -##################################### -# # -# Display Basic Results # -# # -##################################### - - -print "\n\n----------- Access Log Output ------------\n"; -print "\nStart of Logs: $start\n"; -print "End of Logs: $end\n\n"; -if($elapsedDays eq "0"){ - print "Processed Log Time: $elapsedHours Hours, $elapsedMinutes Minutes, $elapsedSeconds Seconds\n\n"; -} else { - print "Processed Log Time: $elapsedDays Days, $elapsedHours Hours, $elapsedMinutes Minutes, $elapsedSeconds Seconds\n\n"; -} - -# -# Check here if we are producing any unqiue reports -# - -if($reportBinds eq "yes"){ - &displayBindReport(); -} - -# -# Continue with standard report -# - -print "Restarts: $restarts\n"; -print "Total Connections: $connectionCount\n"; -print "SSL Connections: $sslconn\n"; -print "Peak Concurrent Connections: $maxsimConnection\n"; -print "Total Operations: $allOps\n"; -print "Total Results: $allResults\n"; -if ($allOps ne "0"){ - print sprintf "Overall Performance: %.1f%\n\n" , ($perf = ($tmp = ($allResults / $allOps)*100) > 100 ? 100.0 : $tmp) ; - } -else { - print "Overall Performance: No Operations to evaluate\n\n"; -} - -$searchStat = sprintf "(%.2f/sec) (%.2f/min)\n",($search / $totalTimeInSecs), $search / ($totalTimeInSecs/60); -$modStat = sprintf "(%.2f/sec) (%.2f/min)\n",$mod / $totalTimeInSecs, $mod/($totalTimeInSecs/60); -$addStat = sprintf "(%.2f/sec) (%.2f/min)\n",$add/$totalTimeInSecs, $add/($totalTimeInSecs/60); -$deleteStat = sprintf "(%.2f/sec) (%.2f/min)\n",$delete/$totalTimeInSecs, $delete/($totalTimeInSecs/60); -$modrdnStat = sprintf "(%.2f/sec) (%.2f/min)\n",$modrdn/$totalTimeInSecs, $modrdn/($totalTimeInSecs/60); -$compareStat = sprintf "(%.2f/sec) (%.2f/min)\n",$compare/$totalTimeInSecs, $compare/($totalTimeInSecs/60); -$bindStat = sprintf "(%.2f/sec) (%.2f/min)\n",$bind/$totalTimeInSecs, $bind/($totalTimeInSecs/60); - -format STDOUT = -Searches: @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< - $search, $searchStat -Modifications: @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< - $mod, $modStat -Adds: @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< - $add, $addStat -Deletes: @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< - $delete, $deleteStat -Mod RDNs: @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< - $modrdn, $modrdnStat -Compares: @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< - $compare, $compareStat -Binds: @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< - $bind $bindStat -. -write STDOUT; - -print "\n"; -print "Proxied Auth Operations: $proxiedAuth\n"; -print "Persistent Searches: $persistent\n"; -print "Internal Operations: $internal\n"; -print "Entry Operations: $entryOp\n"; -print "Extended Operations: $extendedop\n"; -print "Abandoned Requests: $abandon\n"; -print "Smart Referrals Received: $referral\n"; -print "\n"; -print "VLV Operations: $vlv\n"; -print "VLV Unindexed Searches: $vlvnotes\n"; -print "SORT Operations: $sortvlv\n"; -print "\n"; -print "Entire Search Base Queries: $objectclass\n"; -print "Unindexed Searches: $notes\n"; -if ($verb eq "yes" || $usage =~ /u/){ -if ($notes > 0){ - $ns = "1"; - for ($n = 0; $n <= $#notesEtime; $n++){ - @alreadyseenDN = (); - print "\n Unindexed Search #".$ns."\n"; - $ns++; - print " - Date/Time: $notesTime[$n]\n"; - print " - Connection Number: $notesConn[$n]\n"; - print " - Operation Number: $notesOp[$n]\n"; - print " - Etime: $notesEtime[$n]\n"; - print " - Nentries: $notesNentries[$n]\n"; - print " - IP Address: $conn_hash{$notesConn[$n]}\n"; - for ($nn = 0; $nn <= $bc; $nn++){ - if ($notesConn[$n] eq $bindInfo[$nn][1]) { - - ## Here, we check if the bindDN is already printed. - ## If not, we print it and push it to @alreadyseenDN. - ## So, in the beginning, we iterate thru @alreadyseenDN - - for ($j=0, $DNisThere=0; $j <=$#alreadyseenDN; $j++) { - if ($alreadyseenDN[$j] eq $bindInfo[$nn][0]) { - $DNisThere = 1; - } - } - unless ($DNisThere) { - print " - Bind DN: $bindInfo[$nn][0]\n"; - push @alreadyseenDN, $bindInfo[$nn][0]; - } - } - } - for ($nnn = 0; $nnn <= $bcc; $nnn++){ - if ($notesConn[$n] eq $baseInfo[$nnn][1] && $notesOp[$n] eq $baseInfo[$nnn][2]){ - print " - Search Base: $baseInfo[$nnn][0]\n"; - } - } - for ($nnn = 0; $nnn <= $scc; $nnn++){ - if ($notesConn[$n] eq $scopeInfo[$nnn][1] && $notesOp[$n] eq $scopeInfo[$nnn][2]){ - print " - Search Scope: $scopeInfo[$nnn][0]\n"; - } - } - for ($nnn = 0; $nnn <= $fcc; $nnn++){ - if ($notesConn[$n] eq $filterInfo[$nnn][1] && $notesOp[$n] eq $filterInfo[$nnn][2]){ - print " - Search Filter: $filterInfo[$nnn][0]\n"; - } - } - } -} - -} - -print "\n"; -print "FDs Taken: $fdtake\n"; -print "FDs Returned: $fdreturn\n"; -print "Highest FD Taken: $highfd\n\n"; -print "Broken Pipes: $broken\n"; -if ($broken > 0){ - foreach $key (sort { $rc{$b} <=> $rc{$a} } keys %rc) { - if ($rc{$key} > 0){ - if ($conn{$key} eq ""){$conn{$key} = "** Unknown **";} - push @etext, sprintf " - %-4s (%2s) %-40s\n",$rc{$key},$conn{$key},$connmsg{$key -}; - } - } - print @etext; - print "\n"; -} - -print "Connections Reset By Peer: $reset\n"; -if ($reset > 0){ - foreach $key (sort { $src{$b} <=> $src{$a} } keys %src) { - if ($src{$key} > 0){ - if ($conn{$key} eq ""){$conn{$key} = "** Unknown **";} - push @retext, sprintf " - %-4s (%2s) %-40s\n",$src{$key},$conn{$key},$connmsg{$key -}; - } - } - print @retext; - print "\n"; -} - -print "Resource Unavailable: $resource\n"; -if ($resource > 0){ - foreach $key (sort { $rsrc{$b} <=> $rsrc{$a} } keys %rsrc) { - if ($rsrc{$key} > 0){ - if ($conn{$key} eq ""){$conn{$key} = "** Resource Issue **";} - push @rtext, sprintf " - %-4s (%2s) %-40s\n",$rsrc{$key},$conn{$key},$connmsg{$key}; - } - } - print @rtext; -} -print "\n"; -print "Binds: $bind\n"; -print "Unbinds: $unbind\n"; -print "\n LDAP v2 Binds: $version2\n"; -print " LDAP v3 Binds: $version3\n"; -print " SSL Client Binds: $sslClientBind\n"; -print " Failed SSL Client Binds: $sslClientFailed\n"; -print " SASL Binds: $sasl\n"; -if ($sasl > 0){ - foreach $saslb ( sort {$saslmech{$b} <=> $saslmech{$a} } (keys %saslmech) ){ - printf " %-4s %-12s\n",$saslmech{$saslb}, $saslb; - } -} - -print "\n Directory Manager Binds: $dirmgr\n"; -print " Anonymous Binds: $anony\n"; -$other = $bind -($dirmgr + $anony); -print " Other Binds: $other"; - -if ($verb eq "yes" || $usage =~ /y/){ -print "\n\n----- Connection Latency Details -----\n\n"; -print " (in seconds)\t\t<=1\t2\t3\t4-5\t6-10\t11-15\t>15\n"; -print " --------------------------------------------------------------------------\n"; -print " (# of connections)\t"; -for ($i=0; $i <=$#latency; $i++) { - print "$latency[$i]\t"; -} -} - -if ($verb eq "yes" || $usage =~ /p/){ -print "\n\n----- Current Open Connection IDs ----- \n\n"; -for ($i=0; $i <= $#openConnection ; $i++) { - if ($openConnection[$i]) { - print "\t$i\n"; - } -} -} - -################################### -# # -# Display Error Codes # -# # -################################### - -if ($usage =~ /e/i || $verb eq "yes"){ -print "\n\n----- Errors -----\n"; - -%er = sort( {$b <=> $a} %er); -for ($i = 0; $i<98; $i++){ - if ($err[$i] ne "" && $er[$i] >0) { - push @errtext, sprintf "%-8s %12s %-25s","err=$i",$er[$i],$err[$i]; - } -} - -for ($i = 0; $i < $#errtext; $i++){ - - for ($ii = 0; $ii < $#errtext; $ii++){ - $yy="0"; - $zz="0"; - while ($errtext[$ii] =~ /(\w+)\s/g){ - $errornum[$yy]="$1"; - $yy++; - } - while ($errtext[$ii+1] =~ /(\w+)\s/g){ - $errornum2[$zz]="$1"; - $zz++; - } - - if ($errornum2[1] > $errornum[1]){ - $tmp = $errtext[$ii]; - $errtext[$ii] = $errtext[$ii+1]; - $errtext[$ii+1] = $tmp; - } - } -} - -for ($i = 0; $i <= $#errtext; $i++){ - $errtext[$i] =~ s/\n//g; - print "\n" . $errtext[$i]; -} - -} - -#################################### -# # -# Print Failed Logins # -# # -#################################### - -if ($verb eq "yes" || $usage =~ /f/i ){ -if ($bpc > 0){ -print "\n\n----- Top $sizeCount Failed Logins ------\n\n"; - -if ($ds6x eq "true"){ - $eloop = "0"; - foreach $dsbp (sort { $ds6xbadpwd{$b} <=> $ds6xbadpwd{$a} } keys %ds6xbadpwd) { - if ($eloop > $sizeCount){ last; } - printf "%-4s %-40s\n", $ds6xbadpwd{$dsbp}, $dsbp; - } - -} else { -for ($ii =0 ; $ii < $bpc; $ii++){ - for ($i = 0; $i < $bc; $i++){ - if ($badPasswordConn[$ii] eq $bindInfo[$i][1] && $badPasswordOp[$ii] eq $bindInfo[$i][2] ){ - $badPassword{ "$bindInfo[$i][0]" } = $badPassword{ "$bindInfo[$i][0]" } + 1; - } - } -} - -# sort the new list of $badPassword{} - -$bpTotal = "0"; -$bpCount = "0"; -foreach $badpw (sort {$badPassword{$b} <=> $badPassword{$a} } keys %badPassword){ - if ($bpCount > $sizeCount){ last;} - $bpCount++; - $bpTotal = $bpTotal + $badPassword{"$badpw"}; - printf "%-4s %-40s\n", $badPassword{"$badpw"}, $badpw; -} - -print "\nFrom the IP address(s) :\n\n"; -for ($i=0; $i<$bpi; $i++) { - print "\t\t$badPasswordIp[$i]\n"; -} - -if ($bpTotal > $bpc){ - print "\n** Warning : Wrongly reported failed login attempts : ". ($bpTotal - $bpc) . "\n"; -} -} # this ends the if $ds6x = true - -} - -} - - -#################################### -# # -# Print Connection Codes # -# # -#################################### - - -if ($concount > 0){ -if ($usage =~ /c/i || $verb eq "yes"){ - print "\n\n----- Total Connection Codes -----\n\n"; - - foreach $key (sort { $conncount{$b} <=> $conncount{$a} } keys %conncount) { - if ($conncount{$key} > 0){ - push @conntext, sprintf "%-4s %6s %-40s\n",$conn{ $key },$conncount{$key},$connmsg{ $key }; - } - } -print @conntext; -} - -} - -######################################## -# # -# Gather and Process all unique IPs # -# # -######################################## - -if ($usage =~ /i/i || $verb eq "yes"){ -@ipkeys = keys %ip_hash; -@exxCount = keys %exCount; -$ip_count = ($#ipkeys + 1)-($#exxCount + 1); -if ($ip_count > 0){ - print "\n\n----- Top $sizeCount Clients -----\n\n"; - print "Number of Clients: $ip_count\n\n"; - - foreach $key (sort { $ip_hash{$b}{"count"} <=> $ip_hash{$a}{"count"} } keys %ip_hash) { - $exc = "no"; - if ($ccount > $sizeCount){ last;} - $ccount++; - for ($xxx =0; $xxx <= $#exclude; $xxx++){ - if ($exclude[$xxx] eq $key){$exc = "yes";} - } - if ($exc ne "yes"){ - if ($ip_hash{ $key }{"count"} eq ""){$ip_hash{ $key }{"count"} = "*";} - printf "%-6s %-17s\n", $ip_hash{ $key }{"count"}, $key; - } - - if ($exc ne "yes"){ - foreach $code (sort { $ip_hash{ $key }{$b} <=> $ip_hash{ $key }{$a} } keys %{$ip_hash{ $key }}) { - if ($code eq 'count' ) { next; } - printf "\t\t %6s - %3s %s\n", $ip_hash{ $key }{ $code }, $code, $connmsg{ $code }; - } - } - - if ($exc ne "yes"){ print "\n";} - } -} -} - - - -################################### -# # -# Gather All unique Bind DN's # -# # -################################### - -if ($usage =~ /b/i || $verb eq "yes"){ -@bindkeys = keys %bindlist; -$bind_count = $#bindkeys + 1; -if ($bind_count > 0){ - print "\n----- Top $sizeCount Bind DN's -----\n\n"; - print "Number of Unique Bind DN's: $bind_count\n\n"; - - $bindcount = 0; - - foreach $dn (sort { $bindlist{$b} <=> $bindlist{$a} } keys %bindlist) { - if ($bindcount < $sizeCount){ - printf "%-8s %-40s\n", $bindlist{ $dn },$dn; - } - $bindcount++; - } -} - -} - - -######################################### -# # -# Gather and process search bases # -# # -######################################### - -if ($usage =~ /a/i || $verb eq "yes"){ -@basekeys = keys %base; -$base_count = $#basekeys + 1; -if ($base_count > 0){ - print "\n\n----- Top $sizeCount Search Bases -----\n\n"; - print "Number of Unique Search Bases: $base_count\n\n"; - - $basecount = 0; - - foreach $bas (sort { $base{$b} <=> $base{$a} } keys %base) { - if ($basecount < $sizeCount){ - printf "%-8s %-40s\n", $base{ $bas },$bas; - } - $basecount++; - } -} - -} - -######################################### -# # -# Gather and process search filters # -# # -######################################### - -if ($usage =~ /l/ || $verb eq "yes"){ - -@filterkeys = keys %filter; -$filter_count = $#filterkeys + 1; -if ($filter_count > 0){ - print "\n\n----- Top $sizeCount Search Filters -----\n"; - print "\nNumber of Unique Search Filters: $filter_count\n\n"; - - $filtercount = 0; - - foreach $filt (sort { $filter{$b} <=> $filter{$a} } keys %filter){ - if ($filtercount < $sizeCount){ - printf "%-8s %-40s\n", $filter{$filt}, $filt; - } - $filtercount++; - } -} - -} - -######################################### -# # -# Gather and Process the unique etimes # -# # -######################################### - -if ($usage =~ /t/i || $verb eq "yes"){ - # - # print most often etimes - # - print "\n\n----- Top $sizeCount Most Frequent etimes -----\n\n"; - $eloop = 0; - foreach $et (sort { $etime{$b} <=> $etime{$a} } keys %etime) { - if ($eloop == $sizeCount) { last; } - if ($retime ne "2"){ - $first = $et; - $retime = "2"; - } - printf "%-8s %-12s\n", $etime{ $et }, "etime=$et"; - $eloop++; - } - # - # print longest etimes - # - print "\n\n----- Top $sizeCount Longest etimes -----\n\n"; - $eloop = 0; - foreach $et (sort { $b <=> $a } (keys %etime)) { - if ($eloop == $sizeCount) { last; } - printf "%-12s %-10s\n","etime=$et",$etime{ $et }; - $eloop++; - } -} - -####################################### -# # -# Gather and Process unique nentries # -# # -####################################### - - -if ($usage =~ /n/i || $verb eq "yes"){ -print "\n\n----- Top $sizeCount Largest nentries -----\n\n"; -$eloop = 0; -foreach $nentry (sort { $b <=> $a } (keys %nentries)){ - if ($eloop == $sizeCount) { last; } - printf "%-18s %12s\n","nentries=$nentry", $nentries{ $nentry }; - $eloop++; -} - -print "\n\n----- Top $sizeCount Most returned nentries -----\n\n"; -$eloop = 0; -foreach $nentry (sort { $nentries{$b} <=> $nentries{$a} } (keys %nentries)){ - if ($eloop == $sizeCount) { last; } - printf "%-12s %-14s\n", $nentries{ $nentry }, "nentries=$nentry"; - $eloop++; -} - -print "\n"; -} - - -########################################## -# # -# Gather and process extended operations # -# # -########################################## - -if ($usage =~ /x/i || $verb eq "yes"){ - -if ($extendedop > 0){ -print "\n\n----- Extended Operations -----\n\n"; -foreach $oids (sort { $oid{$b} <=> $oid{$a} } (keys %oid) ){ - - if ($oids eq "2.16.840.1.113730.3.5.1"){ $oidmessage = "Transaction Request"} - elsif ($oids eq "2.16.840.1.113730.3.5.2"){ $oidmessage = "Transaction Response"} - elsif ($oids eq "2.16.840.1.113730.3.5.3"){ $oidmessage = "Start Replication Request (incremental update)"} - elsif ($oids eq "2.16.840.1.113730.3.5.4"){ $oidmessage = "Replication Response"} - elsif ($oids eq "2.16.840.1.113730.3.5.5"){ $oidmessage = "End Replication Request (incremental update)"} - elsif ($oids eq "2.16.840.1.113730.3.5.6"){ $oidmessage = "Replication Entry Request"} - elsif ($oids eq "2.16.840.1.113730.3.5.7"){ $oidmessage = "Start Bulk Import"} - elsif ($oids eq "2.16.840.1.113730.3.5.8"){ $oidmessage = "Finished Bulk Import"} - elsif ($oids eq "2.16.840.1.113730.3.6.1"){ $oidmessage = "Incremental Update Replication Protocol"} - elsif ($oids eq "2.16.840.1.113730.3.6.2"){ $oidmessage = "Total Update Replication Protocol (Initialization)"} - elsif ($oids eq "2.16.840.1.113730.3.5.9"){ $oidmessage = "Digest Authentication"} - else {$oidmessage = "Other"} - - printf "%-6s %-23s %-60s\n", $oid{ $oids }, $oids, $oidmessage; -} -} - -} - -############################################ -# -# Print most commonly requested attributes -# -############################################ - -if ($usage =~ /r/i || $verb eq "yes"){ -if ($anyAttrs > 0){ -print "\n\n----- Top $sizeCount Most Requested Attributes -----\n\n"; -$eloop = "0"; -foreach $mostAttr (sort { $attr{$b} <=> $attr{$a} } (keys %attr) ){ - if ($eloop eq $sizeCount){ last; } - printf "%-10s %-19s\n", $attr{$mostAttr}, $mostAttr; - $eloop++; -} -} - -} - -################################# -# -# abandoned operation stats -# -################################# - -if ($usage =~ /g/i || $verb eq "yes"){ -$acTotal = $sconn + $dconn + $mconn + $aconn + $mdconn + $bconn + $ubconn + $econn + $mddconn + $cconn; -if ($verb eq "yes" && $ac > 0 && $acTotal > 0){ - -print "\n\n----- Abandon Request Stats -----\n\n"; - - for ($g = 0; $g < $ac; $g++){ - for ($sc = 0; $sc < $sconn; $sc++){ - if ($srchConn[$sc] eq $targetConn[$g] && $srchOp[$sc] eq $targetOp[$g] ){ - print " - SRCH conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; - } - } - for ($dc = 0; $dc < $dconn; $dc++){ - if ($delConn[$dc] eq $targetConn[$g] && $delOp[$dc] eq $targetOp[$g]){ - print " - DEL conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; - } - } - for ($adc = 0; $adc < $aconn; $adc++){ - if ($addConn[$adc] eq $targetConn[$g] && $addOp[$adc] eq $targetOp[$g]){ - print " - ADD conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; - } - } - for ($mc = 0; $mc < $mconn; $mc++){ - if ($modConn[$mc] eq $targetConn[$g] && $modOp[$mc] eq $targetOp[$g]){ - print " - MOD conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; - } - } - for ($mddc = 0; $mddc < $mddconn; $mddc++){ - if ($moddnConn[$mdc] eq $targetConn[$g] && $moddnOp[$mdc] eq $targetOp[$g]){ - print " - MODDN conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; - } - } - for ($cc = 0; $cc < $cconn; $cc++){ - if ($compConn[$mdc] eq $targetConn[$g] && $compOp[$mdc] eq $targetOp[$g]){ - print " - CMP conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; - } - } - for ($mdc = 0; $mdc < $mdconn; $mdc++){ - if ($modrdnConn[$mdc] eq $targetConn[$g] && $modrdnOp[$mdc] eq $targetOp[$g]){ - print " - MODRDN conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; - } - } - for ($bcb = 0; $bcb < $bconn; $bcb++){ - if ($bindConn[$bcb] eq $targetConn[$g] && $bindOp[$bcb] eq $targetOp[$g]){ - print " - BIND conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; - } - } - for ($ubc = 0; $ubc < $ubconn; $ubc++){ - if ($unbindConn[$ubc] eq $targetConn[$g] && $unbindOp[$ubc] eq $targetOp[$g]){ - print " - UNBIND conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; - } - } - for ($ec = 0; $ec < $econn; $ec++){ - if ($extConn[$ec] eq $targetConn[$g] && $extOp[$ec] eq $targetOp[$g]){ - print " - EXT conn=$targetConn[$g] op=$targetOp[$g] msgid=$msgid[$g] client=$conn_hash{$targetConn[$g]}\n"; - } - } - - - } -} - -} - - - -print "\n"; - -####################################### -# # -# Recommendations # -# # -####################################### - -if ($usage =~ /j/i || $verb eq "yes"){ -print "\n----- Recommendations -----\n"; -$recCount = "1"; -if ($notes > 0){ - print "\n $recCount. You have unindexed searches, this can be caused from a search on an unindexed attribute, or your returned results exceeded the allidsthreshold. Unindexed searches are not recommended. To refuse unindexed searches, switch \'nsslapd-require-index\' to \'on\' under your database entry (e.g. cn=UserRoot,cn=ldbm database,cn=plugins,cn=config).\n"; - $recCount++; - } - -if ($conncount{"T1"} > 0){ - print "\n $recCount. You have some connections that are are being closed by the idletimeout setting. You may want to increase the idletimeout if it is set low.\n"; - $recCount++; - } - -if ($conncount{"T2"} > 0){ - print "\n $recCount. You have some coonections that are being closed by the ioblocktimeout setting. You may want to increase the ioblocktimeout.\n"; - $recCount++; - } - -# compare binds to unbinds, if the difference is more than 30% of the binds, then report a issue - -if (($bind - $unbind) > ($bind*.3)){ - print "\n $recCount. You have a significant difference between binds and unbinds. You may want to investigate this difference.\n"; - $recCount++; - } - -# compare fds taken and return, if the difference is more than 30% report a issue - -if (($fdtaken -$fdreturn) > ($fdtaken*.3)){ - print "\n $recCount. You have a significant difference between file descriptors taken and file descriptors returned. You may want to investigate this difference.\n"; - $recCount++; - } - -if ($dirmgr > ($bind *.2)){ - print "\n $recCount. You have a high number of Directory Manager binds. The Directory Manager account should only be used under certain circumstances. Avoid using this account for client applications.\n"; - $recCount++; - } - -if ($errorck > $errorsucc){ - print "\n $recCount. You have more unsuccessful operations than successful operations. You should investigate this difference.\n"; - $recCount++; - } - -if ($conncount{"U1"} < ($concount - $conncount{"U1"})){ - print "\n $recCount. You have more abnormal connection codes than cleanly closed connections. You may want to investigate this difference.\n"; - $recCount++; - } - -if ($first > 0){ - print "\n $recCount. You have a majority of etimes that are greater than zero, you may want to investigate this performance problem.\n"; - $recCount++; - } - -if ($objectclass > ($search *.25)){ - print "\n $recCount. You have a high number of searches that query the entire search base. Although this is not necessarily bad, it could be resource intensive if the search base contains many entries.\n"; - $recCount++; - } - -if ($recCount == 1){ - print "\nNone.\n"; - } -} - -print "\n"; - -# dispaly usage - -sub displayUsage { - - print "Usage:\n\n"; - - print " ./logconv.pl [-h] [-d|--rootdn <rootDN>] [-s|--sizeLimit <size limit>] [-v|verison] [-Vi|verbose]\n"; - print " [-S|--startTime <start time>] [-E|--endTime <end time>] \n"; - print " [-efcibaltnxrgjuyp] [ access log ... ... ]\n\n"; - - print "- Commandline Switches:\n\n"; - - print " -h, --help help/usage\n"; - print " -d, --rootDN <Directory Managers DN> DEFAULT -> cn=directory manager\n"; - print " -s, --sizeLimit <Number of results to return per catagory> DEFAULT -> 20\n"; - print " -X, --excludeIP <IP address to exclude from connection stats> E.g. Load balancers\n"; - print " -v, --version show version of tool\n"; - print " -S, --startTime <time to begin analyzing logfile from>\n"; - print " E.g. \"[28/Mar/2002:13:14:22 -0800]\"\n"; - print " -E, --endTime <time to stop analyzing logfile>\n"; - print " E.g. \"[28/Mar/2002:13:24:62 -0800]\"\n"; - print " -m, --reportFileSecs <CSV output file - per second stats>\n"; - print " -M, --reportFileMins <CSV output file - per minute stats>\n"; - print " -B, --bind <ALL | ANONYMOUS | \"Actual Bind DN\">\n"; - print " -V, --verbose <enable verbose output - includes all stats listed below>\n"; - print " -[efcibaltnxrgjuyp]\n\n"; - - print " e Error Code stats\n"; - print " f Failed Login Stats\n"; - print " c Connection Code Stats\n"; - print " i Client Stats\n"; - print " b Bind Stats\n"; - print " a Search Base Stats\n"; - print " l Search Filter Stats\n"; - print " t Etime Stats\n"; - print " n Nentries Stats\n"; - print " x Extended Operations\n"; - print " r Most Requested Attribute Stats\n"; - print " g Abandoned Operation Stats\n"; - print " j Recommendations\n"; - print " u Unindexed Search Stats\n"; - print " y Connection Latency Stats\n"; - print " p Open Connection ID Stats\n\n"; - - print " Examples:\n\n"; - - print " ./logconv.pl -s 10 -V /logs/access*\n\n"; - print " ./logconv.pl --rootDN cn=dm /logs/access*\n\n"; - print " ./logconv.pl --sizeLimit 50 -ibgju /logs/access*\n\n"; - print " ./logconv.pl -S \"\[28/Mar/2002:13:14:22 -0800\]\" --endTime \"\[28/Mar/2002:13:50:05 -0800\]\" -e /logs/access*\n\n"; - print " ./logconv.pl -m log-minute-stats-csv.out /logs/access*\n\n"; - print " ./logconv.pl -B ANONYMOUS /logs/access*\n\n"; - print " ./logconv.pl -B \"uid=mreynolds,dc=example,dc=com\" /logs/access*\n\n"; - - exit 1; -} - -###################################################### -# -# Parsing Routines That Do The Actual Parsing Work -# -###################################################### - -sub -parseLine { - if($reportBinds eq "yes"){ - &parseLineBind(); - } else { - &parseLineNormal(); - } -} - -sub -parseLineBind { - $ff++; - $iff++; - local $_ = $tline; - - if ($iff >= $limit){ - print STDERR sprintf" %10s Lines Processed\n",$ff; - $iff="0"; - } - - # skip blank lines - return if $_ =~ /^\s/; - - if($firstFile eq "1" && $_ =~ /^\[/){ - $start = $_; - if ($start =~ / *([0-9a-z:\/]+)/i){$start=$1;} - $firstFile = "0"; - } - - if ($endFlag != 1 && $_ =~ /^\[/ && $_ =~ / *([0-9a-z:\/]+)/i){ - $end =$1; - } - - if ($startTime && !$startFlag) { - if (index($_, $startTime) == 0) { - $startFlag = 1; - ($start) = $startTime =~ /\D*(\S*)/; - } else { - return; - } - } - - if ($endTime && !$endFlag) { - if (index($_, $endTime) == 0) { - $endFlag = 1; - ($end) = $endTime =~ /\D*(\S*)/; - } - } - - if ($_ =~ /connection from *([0-9A-Fa-f\.\:]+)/i ) { - for ($excl =0; $excl <= $#exclude; $excl++){ - if ($exclude[$excl] eq $1){ - $skip = "yes"; - last; - } - } - if ($skip eq "yes"){ - return ; - } - $ip = $1; - if ($_ =~ /conn= *([0-9]+)/i ){ - $connList{$ip} = $connList{$ip} . " $1 "; - } - return; - } - - if (/ BIND/ && $_ =~ /dn=\"(.*)\" method/i ){ - if ($1 eq ""){ - $dn = "Anonymous"; - } else { - $dn = $1; - $dn =~ tr/A-Z/a-z/; - } - - if($bindReportDN ne ""){ - if($dn ne $bindReportDN){ - # We are not looking for this DN, skip it - return; - } - } - - $bindReport{$dn}{"binds"}++; - if ($bindReport{$dn}{"binds"} eq 1){ - # For hashes we need to init the counters - $bindReport{$dn}{"srch"} = 0; - $bindReport{$dn}{"add"} = 0; - $bindReport{$dn}{"mod"} = 0; - $bindReport{$dn}{"del"} = 0; - $bindReport{$dn}{"cmp"} = 0; - $bindReport{$dn}{"ext"} = 0; - $bindReport{$dn}{"modrdn"} = 0; - $bindReport{$dn}{"failedBind"} = 0; - } - - if ($_ =~ /conn= *([0-9]+)/i) { - $bindReport{$dn}{"conn"} = $bindReport{$dn}{"conn"} . " $1 "; - } - return; - } - - if (/ RESULT err=49 /){ - processOpForBindReport("failedBind",$tline); - } - - if (/ SRCH base=/){ - processOpForBindReport("srch",$tline); - } elsif (/ ADD dn=/){ - processOpForBindReport("add",$tline); - } elsif (/ MOD dn=/){ - processOpForBindReport("mod",$tline); - } elsif (/ DEL dn=/){ - processOpForBindReport("del",$tline); - } elsif (/ MODRDN dn=/){ - processOpForBindReport("modrdn",$tline); - } elsif (/ CMP dn=/){ - processOpForBindReport("cmp",$tline); - } elsif (/ EXT oid=/){ - processOpForBindReport("ext",$tline); - } - -} - -sub -processOpForBindReport -{ - $op = @_[0]; - $data = @_[1]; - - if ($data =~ /conn= *([0-9]+)/i) { - foreach $dn (keys %bindReport){ - if ($bindReport{$dn}{"conn"} =~ / $1 /){ - $bindDN = $dn; - $bindReport{$bindDN}{$op}++; - return; - } - } - } -} - -sub parseLineNormal { -local $_ = $tline; - -# lines starting blank are restart -return if $_ =~ /^\s/; - -if($firstFile eq "1" && $_ =~ /^\[/){ - # if we are using startTime & endTime, this will get overwritten, which is ok - $start = $_; - if ($start =~ / *([0-9a-z:\/]+)/i){$start=$1;} - $firstFile = "0"; -} - -if ($endFlag != 1 && $_ =~ /^\[/ && $_ =~ / *([0-9a-z:\/]+)/i){$end =$1;} - -if ($startTime && !$startFlag) { - if (index($_, $startTime) == 0) { - $startFlag = 1; - ($start) = $startTime =~ /\D*(\S*)/; - } else { - return; - } -} - -if ($endTime && !$endFlag) { - if (index($_, $endTime) == 0) { - $endFlag = 1; - ($end) = $endTime =~ /\D*(\S*)/; - } -} - -$ff++; -$iff++; -if ($iff >= $limit){ print STDERR sprintf" %10s Lines Processed\n",$ff; $iff="0";} - -# Additional performance stats -($time, $tzone) = split (' ', $_); -if ($time ne $last_tm) -{ - $last_tm = $time; - - $time =~ s/\[//; - $tzone =~ s/\].*//; - - if($tzone ne $lastzone) - { - # tz offset change - $lastzone=$tzone; - ($sign,$hr,$min) = $tzone =~ m/(?)(\d\d)(\d\d)/; - $tzoff = $hr*3600 + $min*60; - $tzoff *= -1 - if $sign eq '-'; - # to be subtracted from converted values. - } - - ($date, $hr, $min, $sec) = split (':', $time); - ($day, $mon, $yr) = split ('/', $date); - $newmin = timegm(0, $min, $hours, $day, $monthname{$mon}, $yr) - $tzoff; - $gmtime = $newmin + $sec; - print_stats_block( $s_stats ); - reset_stats_block( $s_stats, $gmtime, $time.' '.$tzone ); - if ($newmin != $last_min) - { - print_stats_block( $m_stats ); - $time =~ s/\d\d$/00/; - reset_stats_block( $m_stats, $newmin, $time.' '.$tzone ); - $last_min = $newmin; - } -} - -if (m/ RESULT err/){$allResults++;inc_stats('results',$s_stats,$m_stats);} -if (m/ SRCH/){ - $search++; - inc_stats('srch',$s_stats,$m_stats); - if ($_ =~ / attrs=\"(.*)\"/i){ - $anyAttrs++; - $attrs = $1 . " "; - while ($attrs =~ /(\S+)\s/g){ - $attr{$1}++; - } - } - if (/ attrs=ALL/){ - $attr{"All Attributes"}++; - $anyAttrs++; - } - - if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ $srchConn[$sconn] = $1;} - if ($_ =~ /op= *([0-9]+)/i){ $srchOp[$sconn] = $1;} - $sconn++; - } -} -if (m/ DEL/){ - $delete++; - inc_stats('del',$s_stats,$m_stats); - if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ $delConn[$dconn] = $1;} - if ($_ =~ /op= *([0-9]+)/i){ $delOp[$dconn] = $1;} - $dconn++; - } -} -if (m/ MOD dn=/){ - $mod++; - inc_stats('mod',$s_stats,$m_stats); - if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ $modConn[$mconn] = $1;} - if ($_ =~ /op= *([0-9]+)/i){ $modOp[$mconn] = $1; } - $mconn++; - } -} -if (m/ MODDN dn=/){ - $moddn++; - inc_stats('moddn',$s_stats,$m_stats); - if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ $moddnConn[$mconn] = $1;} - if ($_ =~ /op= *([0-9]+)/i){ $moddnOp[$mconn] = $1; } - $mddconn++; - } -} -if (m/ ADD/){ - $add++; - inc_stats('add',$s_stats,$m_stats); - if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ $addConn[$aconn] = $1; } - if ($_ =~ /op= *([0-9]+)/i){ $addOp[$aconn] = $1; } - $aconn++; - } -} -if (m/ MODRDN/){ - $modrdn++; - inc_stats('modrdn',$s_stats,$m_stats); - if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ $modrdnConn[$mdconn] = $1; } - if ($_ =~ /op= *([0-9]+)/i){ $modrdnOp[$mdconn] = $1; } - $mdconn++; - } -} -if (m/ CMP dn=/){ - $compare++; - inc_stats('cmp',$s_stats,$m_stats); - if ($verb eq "yes" || $usage =~ /g/i){ - if ($_ =~ /conn= *([0-9]+)/i){ $compConn[$dconn] = $1;} - if ($_ =~ /op= *([0-9]+)/i){ $compOp[$dconn] = $1;} - $cconn++; - } -} -if (m/ ABANDON /){ - $abandon++; - inc_stats('abandon',$s_stats,$m_stats); - $allResults++; - if ($_ =~ /targetop= *([0-9a-zA-Z]+)/i ){ - $targetOp[$ac] = $1; - if ($_ =~ /conn= *([0-9]+)/i){ $targetConn[$ac] = $1; } - if ($_ =~ /msgid= *([0-9]+)/i){ $msgid[$ac] = $1;} - $ac++; - } -} -if (m/ VLV /){ - if ($_ =~ /conn= *([0-9]+)/i){ $vlvconn[$vlv] = $1;} - if ($_ =~ /op= *([0-9]+)/i){ $vlvop[$vlv] = $1;} - $vlv++; -} -if (m/ authzid=/){ - $proxiedAuth++; -} -if (m/ SORT /){$sortvlv++} -if (m/ version=2/){$version2++} -if (m/ version=3/){$version3++} -if (m/ conn=1 fd=/){$restarts++} -if (m/ SSL connection from/){$sslconn++;} -if (m/ connection from/){ - $exc = "no"; - if ($_ =~ /connection from *([0-9A-Fa-f\.\:]+)/i ){ - for ($xxx =0; $xxx <= $#exclude; $xxx++){ - if ($exclude[$xxx] eq $1){$exc = "yes";} - } - if ($exc ne "yes"){ $connectionCount++;} - } - $simConnection++; - if ($simConnection > $maxsimConnection) { - $maxsimConnection = $simConnection; - } - - ($connID) = $_ =~ /conn=(\d*)\s/; - $openConnection[$connID]++; - ($time, $tzone) = split (' ', $_); - ($date, $hr, $min, $sec) = split (':', $time); - ($day, $mon, $yr) = split ('/', $date); - $day =~ s/\[//; - $start_time_of_connection[$connID] = timegm($sec, $min, $hours, $day, $monthname{$mon}, $yr); -} - -if (m/ SSL client bound as /){$sslClientBind++;} -if (m/ SSL failed to map client certificate to LDAP DN/){$sslClientFailed++;} -if (m/ fd=/ && m/slot=/){$fdtake++} -if (m/ fd=/ && m/closed/){ - $fdreturn++; - $simConnection--; - - ($connID) = $_ =~ /conn=(\d*)\s/; - $openConnection[$connID]--; - $end_time_of_connection[$connID] = $gmtime; - $diff = $end_time_of_connection[$connID] - $start_time_of_connection[$connID]; - $start_time_of_connection[$connID] = $end_time_of_connection[$connID] = 0; - if ($diff <= 1) { $latency[0]++;} - if ($diff == 2) { $latency[1]++;} - if ($diff == 3) { $latency[2]++;} - if ($diff >= 4 && $diff <=5 ) { $latency[3]++;} - if ($diff >= 6 && $diff <=10 ) { $latency[4]++;} - if ($diff >= 11 && $diff <=15 ) { $latency[5]++;} - if ($diff >= 16) { $latency[6] ++;} -} -if (m/ BIND/){ - $bind++; - inc_stats('bind',$s_stats,$m_stats); - if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ $bindConn[$bconn] = $1; } - if ($_ =~ /op= *([0-9]+)/i){ $bindOp[$bconn] = $1; } - $bconn++; - } -} -if (m/ BIND/ && m/$manager/i){$dirmgr++} -if (m/ BIND/ && m/dn=""/){$anony++; $bindlist{"Anonymous Binds"}++;inc_stats('anonbind',$s_stats,$m_stats);} -if (m/ UNBIND/){ - $unbind++; - if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ $unbindConn[$ubconn] = $1; } - if ($_ =~ /op= *([0-9]+)/i){ $unbindOp[$ubconn] = $1; } - $ubconn++; - } -} -if (m/ notes=U/){ - if ($_ =~ /conn= *([0-9]+)/i){ - $con = $1; - if ($_ =~ /op= *([0-9]+)/i){ $op = $1;} - } - for ($i=0; $i <= $vlv;$i++){ - if ($vlvconn[$i] eq $con && $vlvop[$i] eq $op){ $vlvnotes++; $v="1";} - } - if($v ne "1"){ - # We don't want to record vlv unindexed searches for our regular "bad" - # unindexed search stat, as VLV unindexed searches aren't that bad - $notes++; - inc_stats('notesu',$s_stats,$m_stats); - } - if ($usage =~ /u/ || $verb eq "yes"){ - if ($v eq "0" ){ - if ($_ =~ /etime= *([0-9.]+)/i ) { - $notesEtime[$vet]=$1; - $vet++; - } - if ($_ =~ /conn= *([0-9]+)/i){ - $notesConn[$nc]=$1; - $nc++; - } - if ($_ =~ /op= *([0-9]+)/i){ - $notesOp[$no]=$1; - $no++; - } - if ($_ =~ / *([0-9a-z:\/]+)/i){ - $notesTime[$nt] = $1; - $nt++; - } - } - if ($_ =~ /nentries= *([0-9]+)/i ){ - $notesNentries[$nent] = $1; - $nent++; - } - } - $v = "0"; -} - -if (m/ closed error 32/){ - $broken++; - if (m/- T1/){ $rc{"T1"}++ } - elsif (m/- T2/){ $rc{"T2"}++ } - elsif (m/- A1/){ $rc{"A1"}++ } - elsif (m/- B1/){ $rc{"B1"}++ } - elsif (m/- B4/){ $rc{"B4"}++ } - elsif (m/- B2/){ $rc{"B2"}++ } - elsif (m/- B3/){ $rc{"B3"}++ } - elsif (m/- R1/){ $rc{"R1"}++ } - elsif (m/- P1/){ $rc{"P1"}++ } - elsif (m/- P1/){ $rc{"P2"}++ } - elsif (m/- U1/){ $rc{"U1"}++ } - else { $rc{"other"}++; } -} -if (m/ closed error 131/ || m/ closed error -5961/){ - $reset++; - if (m/- T1/){ $src{"T1"}++ } - elsif (m/- T2/){ $src{"T2"}++ } - elsif (m/- A1/){ $src{"A1"}++ } - elsif (m/- B1/){ $src{"B1"}++ } - elsif (m/- B4/){ $src{"B4"}++ } - elsif (m/- B2/){ $src{"B2"}++ } - elsif (m/- B3/){ $src{"B3"}++ } - elsif (m/- R1/){ $src{"R1"}++ } - elsif (m/- P1/){ $src{"P1"}++ } - elsif (m/- P1/){ $src{"P2"}++ } - elsif (m/- U1/){ $src{"U1"}++ } - else { $src{"other"}++ } -} - -if (m/ closed error 11/){ - $resource++; - if (m/- T1/){ $rsrc{"T1"}++ } - elsif (m/- T2/){ $rsrc{"T2"}++ } - elsif (m/- A1/){ $rsrc{"A1"}++ } - elsif (m/- B1/){ $rsrc{"B1"}++ } - elsif (m/- B4/){ $rsrc{"B4"}++ } - elsif (m/- B2/){ $rsrc{"B2"}++ } - elsif (m/- B3/){ $rsrc{"B3"}++ } - elsif (m/- R1/){ $rsrc{"R1"}++ } - elsif (m/- P1/){ $rsrc{"P1"}++ } - elsif (m/- P1/){ $rsrc{"P2"}++ } - elsif (m/- U1/){ $rsrc{"U1"}++ } - else { $rsrc{"other"}++ } -} - -if ($usage =~ /g/ || $usage =~ /c/ || $usage =~ /i/ || $verb eq "yes"){ - -$exc = "no"; - -if ($_ =~ /connection from *([0-9A-fa-f\.\:]+)/i ) { - for ($xxx = 0; $xxx <= $#exclude; $xxx++){ - if ($1 eq $exclude[$xxx]){ - $exc = "yes"; - $exCount{$1}++; - } - } - $ip = $1; - $ip_hash{$ip}{"count"}++; - if ($_ =~ /conn= *([0-9]+)/i ){ - if ($exc ne "yes"){ $ip_hash2{$ip} = sprintf "%-12s %18s\n",$1,$ip;} - $conn_hash{$1} = $ip; - - } - -} -if (m/- A1/){ - if ($_ =~ /conn= *([0-9]+)/i) { - $exc = "no"; - $ip = $conn_hash{$1}; - if ($ip eq ""){$ip = "Unknown Host";} - for ($xxx = 0; $xxx <= $#exclude; $xxx++){ - if ($ip eq $exclude[$xxx]){$exc = "yes";} - } - if ($exc ne "yes"){ - $ip_hash{$ip}{"A1"}++; - $conncount{"A1"}++; - $concount++; - } - } -} -if (m/- B1/){ - if ($_ =~ /conn= *([0-9]+)/i) { - $exc = "no"; - $ip = $conn_hash{$1}; - if ($ip eq ""){$ip = "Unknown Host";} - for ($xxx = 0; $xxx <= $#exclude; $xxx++){ - if ($ip eq $exclude[$xxx]){$exc = "yes";} - } - if ($exc ne "yes"){ - $ip_hash{$ip}{"B1"}++; - $conncount{"B1"}++; - $concount++; - } - } -} -if (m/- B4/){ - if ($_ =~ /conn= *([0-9]+)/i) { - $exc = "no"; - $ip = $conn_hash{$1}; - if ($ip eq ""){$ip = "Unknown Host";} - for ($xxx = 0; $xxx <= $#exclude; $xxx++){ - if ($ip eq $exclude[$xxx]){$exc = "yes";} - } - if ($exc ne "yes"){ - $ip_hash{$ip}{"B4"}++; - $conncount{"B4"}++; - $concount++; - } - } -} -if (m/- T1/){ - if ($_ =~ /conn= *([0-9]+)/i) { - $exc = "no"; - $ip = $conn_hash{$1}; - if ($ip eq ""){$ip = "Unknown Host";} - for ($xxx = 0; $xxx <= $#exclude; $xxx++){ - if ($ip eq $exclude[$xxx]){$exc = "yes";} - } - if ($exc ne "yes"){ - $ip_hash{$ip}{"T1"}++; - $conncount{"T1"}++; - $concount++; - } - } -} -if (m/- T2/){ - if ($_ =~ /conn= *([0-9]+)/i) { - $exc = "no"; - $ip = $conn_hash{$1}; - if ($ip eq ""){$ip = "Unknown Host";} - for ($xxx = 0; $xxx <= $#exclude; $xxx++){ - if ($ip eq $exclude[$xxx]){$exc = "yes";} - } - if ($exc ne "yes"){ - $ip_hash{$ip}{"T2"}++; - $conncount{"T2"}++; - $concount++; - } - } -} -if (m/- B2/){ - if ($_ =~ /conn= *([0-9]+)/i) { - $exc = "no"; - $ip = $conn_hash{$1}; - if ($ip eq ""){$ip = "Unknown Host";} - for ($xxx = 0; $xxx <= $#exclude; $xxx++){ - if ($ip eq $exclude[$xxx]){$exc = "yes";} - } - if ($exc ne "yes"){ - $ip_hash{$ip}{"B2"}++; - $conncount{"B2"}++; - $concount++; - } - } -} -if (m/- B2/){ - if ($_ =~ /conn= *([0-9]+)/i) { - $exc = "no"; - $ip = $conn_hash{$1}; - if ($ip eq ""){$ip = "Unknown Host";} - for ($xxx = 0; $xxx <= $#exclude; $xxx++){ - if ($ip eq $exclude[$xxx]){$exc = "yes";} - } - if ($exc ne "yes"){ - $ip_hash{$ip}{"B2"}++; - $conncount{"B2"}++; - $concount++; - } - } -} -if (m/- B3/){ - if ($_ =~ /conn= *([0-9]+)/i) { - $exc = "no"; - $ip = $conn_hash{$1}; - if ($ip eq ""){$ip = "Unknown Host";} - for ($xxx = 0; $xxx <= $#exclude; $xxx++){ - if ($ip eq $exclude[$xxx]){$exc = "yes";} - } - if ($exc ne "yes"){ - $ip_hash{$ip}{"B3"}++; - $conncount{"B3"}++; - $concount++; - } - } -} -if (m/- R1/){ - if ($_ =~ /conn= *([0-9]+)/i) { - $exc = "no"; - $ip = $conn_hash{$1}; - if ($ip eq ""){$ip = "Unknown Host";} - for ($xxx = 0; $xxx <= $#exclude; $xxx++){ - if ($ip eq $exclude[$xxx]){$exc = "yes";} - } - if ($exc ne "yes"){ - $ip_hash{$ip}{"R1"}++; - $conncount{"R1"}++; - $concount++; - } - } -} -if (m/- P1/){ - if ($_ =~ /conn= *([0-9]+)/i) { - $exc = "no"; - $ip = $conn_hash{$1}; - if ($ip eq ""){$ip = "Unknown Host";} - for ($xxx = 0; $xxx <= $#exclude; $xxx++){ - if ($ip eq $exclude[$xxx]){$exc = "yes";} - } - if ($exc ne "yes"){ - $ip_hash{$ip}{"P1"}++; - $conncount{"P1"}++; - $concount++; - } - } -} -if (m/- P2/){ - if ($_ =~ /conn= *([0-9]+)/i) { - $exc = "no"; - $ip = $conn_hash{$1}; - if ($ip eq ""){$ip = "Unknown Host";} - for ($xxx = 0; $xxx <= $#exclude; $xxx++){ - if ($ip eq $exclude[$xxx]){$exc = "yes";} - } - if ($exc ne "yes"){ - $ip_hash{$ip}{"P2"}++; - $conncount{"P2"}++; - $concount++; - } - } -} -if (m/- U1/){ - if ($_ =~ /conn= *([0-9]+)/i) { - $exc = "no"; - $ip = $conn_hash{$1}; - if ($ip eq ""){$ip = "Unknown Host";} - for ($xxx = 0; $xxx <= $#exclude; $xxx++){ - if ($ip eq $exclude[$xxx]){$exc = "yes";} - } - if ($exc ne "yes"){ - $ip_hash{$ip}{"U1"}++; - $conncount{"U1"}++; - $concount++; - } - } - -} - -} -if ($_ =~ /err= *([0-9]+)/i){ - $er[$1]++; - if ($1 ne "0"){ $errorck++;} - else { $errorsucc++;} -} -if ($_ =~ /etime= *([0-9.]+)/ ) { $etime{$1}++;} -if ($_ =~ / tag=101 nentries= *([0-9]+)/i ) {$nentries{$1}++} -if ($_ =~ / tag=111 nentries= *([0-9]+)/i ) {$nentries{$1}++} -if ($_ =~ / tag=100 nentries= *([0-9]+)/i ) {$nentries{$1}++} -if ($_ =~ / tag=115 nentries= *([0-9]+)/i ) {$nentries{$1}++} -if (m/objectclass=\*/i || m/objectclass=top/i ){ - if (m/ scope=2 /){ $objectclass++;} -} - -if (m/ EXT oid=/){ - $extendedop++; - if ($_ =~ /oid=\" *([0-9\.]+)/i ){ $oid{$1}++; } - if ($verb eq "yes"){ - if ($_ =~ /conn= *([0-9]+)/i){ $extConn[$econn] = $1; } - if ($_ =~ /op= *([0-9]+)/i){ $extOp[$econn] = $1; } - $econn++; - } -} - -if (m/ BIND/ && $_ =~ /dn=\"(.*)\" method/i ){ - if ($1 ne ""){ - $tmpp = $1; - $tmpp =~ tr/A-Z/a-z/; - $bindlist{$tmpp} = $bindlist{$tmpp} + 1; - - $bindInfo[$bc][0] = $tmpp; - if ($_ =~ /conn= *([0-9]+)/i) { $bindInfo[$bc][1] = $1; } - if ($_ =~ /op= *([0-9]+)/i) { $bindInfo[$bc][2] = $1; } - $bc++; - } -} - -if ($usage =~ /l/ || $verb eq "yes"){ - if (/ SRCH / && / attrs=/ && $_ =~ /filter=\"(.*)\" /i ){ - $tmpp = $1; - $tmpp =~ tr/A-Z/a-z/; - $tmpp =~ s/\\22/\"/g; - $filter{$tmpp} = $filter{$tmpp} + 1; - $filterInfo[$fcc][0] = $tmpp; - if ($_ =~ /conn= *([0-9]+)/i) { $filterInfo[$fcc][1] = $1; } - if ($_ =~ /op= *([0-9]+)/i) { $filterInfo[$fcc][2] = $1; } - $fcc++; - } elsif (/ SRCH / && $_ =~ /filter=\"(.*)\"/i){ - $tmpp = $1; - $tmpp =~ tr/A-Z/a-z/; - $tmpp =~ s/\\22/\"/g; - $filter{$tmpp} = $filter{$tmpp} + 1; - $filterInfo[$fcc][0] = $tmpp; - if ($_ =~ /conn= *([0-9]+)/i) { $filterInfo[$fcc][1] = $1; } - if ($_ =~ /op= *([0-9]+)/i) { $filterInfo[$fcc][2] = $1; } - $fcc++; - } -} - -if ($usage =~ /a/ || $verb eq "yes"){ - if (/ SRCH / && $_ =~ /base=\"(.*)\" scope/i ){ - if ($1 eq ""){ - $tmpp = "Root DSE"; - } else { - $tmpp = $1; - } - $tmpp =~ tr/A-Z/a-z/; - $base{$tmpp} = $base{$tmpp} + 1; - - # - # grab the search bases & scope for potential unindexed searches - # - $baseInfo[$bcc][0] = $tmpp; - if ($_ =~ /scope= *([0-9]+)/i) { - $scopeInfo[$scc][0] = $1; - } - if ($_ =~ /conn= *([0-9]+)/i) { - $baseInfo[$bcc][1] = $1; - $scopeInfo[$scc][1] = $1; - } - if ($_ =~ /op= *([0-9]+)/i) { - $baseInfo[$bcc][2] = $1; - $scopeInfo[$scc][2] = $1; - } - $bcc++; - $scc++; - - } -} - -if ($_ =~ /fd= *([0-9]+)/i ) { - $fds[$fdds] = $1; - if ($fds[$fdds] > $highfd) {$highfd = $fds[$fdds];} - $fdds++; -} - - -if ($usage =~ /f/ || $verb eq "yes"){ - if (/ err=49 tag=/ && / dn=\"/){ - if ($_ =~ /dn=\"(.*)\"/i ){ - $ds6xbadpwd{$1}++; - } - $ds6x = "true"; - $bpc++; - - } elsif (/ err=49 tag=/ ){ - if ($_ =~ /conn= *([0-9]+)/i ){ - $badPasswordConn[$bpc] = $1; - $bpc++; - } - if ($_ =~ /op= *([0-9]+)/i ){ - $badPasswordOp[$bpo] = $1; - $bpo++; - } - $badPasswordIp[$bpi] = $ip; - $bpi++; - } -} - -if (/ BIND / && /method=sasl/i){ - $sasl++; - if ($_ =~ /mech=(.*)/i ){ - $saslmech{$1}++; - } -} - -if (/ conn=Internal op=-1 / && !/ RESULT err=/){ $internal++; } - -if (/ ENTRY dn=/ ){ $entryOp++; } - -if (/ conn=/ && /op=/ && / REFERRAL/){ $referral++; } - -if (/ options=persistent/){$persistent++;} - -} - - ####################################### - # # # - # +# CSV Helper Routines # - # +# # -####################################### -# # -# To convert the CSV to chart in OO # -# # -# * Select active rows and columns # -# * Insert -> Chart # -# * Chart type "XY (Scatter)" # -# * sub-type "Lines Only" # -# * select "Sort by X values" # -# * "Next" # -# * select "Data series in columns" # -# * select "First row as label" # -# * select "First column as label" # -# * "Next" # -# * "Next" # -# * "Finish" # -# # -####################################### - -sub -reset_stats_block -{ - my $stats = shift; - - $stats->{'last'} = shift || 0; - $stats->{'last_str'} = shift || ''; - - $stats->{'results'}=0; - $stats->{'srch'}=0; - $stats->{'add'}=0; - $stats->{'mod'}=0; - $stats->{'modrdn'}=0; - $stats->{'moddn'}=0; - $stats->{'cmp'}=0; - $stats->{'del'}=0; - $stats->{'abandon'}=0; - $stats->{'conns'}=0; - $stats->{'sslconns'}=0; - $stats->{'bind'}=0; - $stats->{'anonbind'}=0; - $stats->{'unbind'}=0; - $stats->{'notesu'}=0; - return; -} - -sub -new_stats_block -{ - my $name = shift || ''; - my $stats = { - 'active' => 0, - }; - - if ($name) - { - $stats->{'filename'} = $name; - $stats->{'fh'} = new IO::File; - $stats->{'active'} = open($stats->{'fh'},">$name"); - } - - reset_stats_block( $stats ); - return $stats; -} - -sub -print_stats_block -{ - foreach my $stats( @_ ) - { - if ($stats->{'active'}) - { - if ($stats->{'last'}) - { - $stats->{'fh'}->print( - join(',', - $stats->{'last_str'}, - $stats->{'last'}, - $stats->{'results'}, - $stats->{'srch'}, - $stats->{'add'}, - $stats->{'mod'}, - $stats->{'modrdn'}, - $stats->{'moddn'}, - $stats->{'cmp'}, - $stats->{'del'}, - $stats->{'abandon'}, - $stats->{'conns'}, - $stats->{'sslconns'}, - $stats->{'bind'}, - $stats->{'anonbind'}, - $stats->{'unbind'}, - $stats->{'notesu'} ), - "\n" ); - }else - { - $stats->{'fh'}->print( - "Time,time_t,Results,Search,Add,Mod,Modrdn,Delete,Abandon,". - "Connections,SSL Conns,Bind,Anon Bind,Unbind,Unindexed\n" - ); - } - } - } - return; -} - -sub -inc_stats -{ - my $n = shift; - foreach(@_) - { - $_->{$n}++ - if exists $_->{$n}; - } - return; -} - -sub -displayBindReport -{ - # - # Loop for each DN - sort alphabetically - # - # Display all the IP addresses, then counts of all the operations it did - # - - print "\nBind Report\n"; - print "====================================================================\n\n"; - foreach $bindDN (sort { $bindReport{$a} <=> $bindReport{$b} } keys %bindReport) { - print("Bind DN: $bindDN\n"); - print("--------------------------------------------------------------------\n"); - print(" Client Addresses:\n\n"); - &printClients($bindReport{$bindDN}{"conn"}); - print("\n Operations Performed:\n\n"); - &printOpStats($bindDN); - print("\n"); - } - print "Done.\n"; - exit (0); -} - -sub -printClients -{ - @bindConns = &cleanConns(split(' ', @_[0])); - $IPcount = "1"; - - foreach $ip ( keys %connList ){ # Loop over all the IP addresses - foreach $bc (@bindConns){ # Loop over each bind conn number and compare it - if($connList{$ip} =~ / $bc /){ - print(" [$IPcount] $ip\n"); - $IPcount++; - last; - } - } - } -} - -sub -cleanConns -{ - @dirtyConns = @_; - $#cleanConns = -1; - $c = 0; - - for ($i = 0; $i <=$#dirtyConns; $i++){ - if($dirtyConns[$i] ne ""){ - $cleanConns[$c++] = $dirtyConns[$i]; - } - } - return @cleanConns; -} - -sub -printOpStats -{ - $dn = @_[0]; - - if( $bindReport{$dn}{"failedBind"} eq "0" ){ - print(" Binds: " . $bindReport{$dn}{"binds"} . "\n"); - } else { - print(" Binds: " . $bindReport{$dn}{"binds"} . " (Invalid Credentials: " . $bindReport{$dn}{"failedBind"} . ")\n"); - } - print(" Searches: " . $bindReport{$dn}{"srch"} . "\n"); - print(" Modifies: " . $bindReport{$dn}{"mod"} . "\n"); - print(" Adds: " . $bindReport{$dn}{"add"} . "\n"); - print(" Deletes: " . $bindReport{$dn}{"del"} . "\n"); - print(" Compares: " . $bindReport{$dn}{"cmp"} . "\n"); - print(" ModRDNs: " . $bindReport{$dn}{"modrdn"} . "\n"); - print(" Ext Ops: " . $bindReport{$dn}{"ext"} . "\n\n"); -} - -####################################### -# # -# The End # -# # -####################################### - commit 9f0c9392ac7e27193831098f1efd4b7047432cd7 Author: Mark Reynolds <mreynolds(a)redhat.com&gt; Date: Tue Jan 8 13:47:25 2013 -0500 Ticket 471 - logconv.pl tool removes the access logs contents if "-M" is not correctly used Bug Description: If you fail to specifiy the output file, the access log is overwritten. Fix Description: Check if the proper files have been provbided to the script, and report error otherwise. https://fedorahosted.org/389/ticket/471 Reviewed by: nhosoi(Thanks!) (cherry picked from commit 89fb3420c9137cb2c9813fec527b89fec3b6d63e) diff --git a/ldap/admin/src/logconv.pl b/ldap/admin/src/logconv.pl index f5342af..0ec596a 100755 --- a/ldap/admin/src/logconv.pl +++ b/ldap/admin/src/logconv.pl @@ -71,6 +71,7 @@ $endFlag = 0; $endTime = 0; $s_stats = new_stats_block( ); $m_stats = new_stats_block( ); +$report_opt = ""; GetOptions( 'd|rootDN=s' => \$manager, @@ -81,8 +82,8 @@ GetOptions( 'S|startTime=s' => \$startTime, 'E|endTime=s' => \$endTime, 'B|bind=s' => sub { $reportBinds = "yes"; $bindReportDN=($_[1]) }, - 'm|reportFileSecs=s' => sub { my ($opt,$value) = @_; $s_stats = new_stats_block($value); }, - 'M|reportFileMins=s' => sub { my ($opt,$value) = @_; $m_stats = new_stats_block($value); }, + 'm|reportFileSecs=s' => sub { my ($opt,$value) = @_; $s_stats = new_stats_block($value); $report_opt = "-m"; }, + 'M|reportFileMins=s' => sub { my ($opt,$value) = @_; $m_stats = new_stats_block($value); $report_opt = "-M"; }, 'h|help' => sub { displayUsage() }, # usage options '-efcibaltnxgjuiryp' 'e' => sub { $usage = $usage . "e"; }, @@ -132,6 +133,15 @@ while($sn <= $#ARGV){ $sn++; } +if($fc == 0){ + if($report_opt ne ""){ + print "Usage error for option $report_opt, either the output file or access log is missing!\n\n"; + } else { + print "There are no access logs specified!\n\n"; + } + exit 1; +} + if ($sizeCount eq "all"){$sizeCount = "100000";} ####################################### @@ -261,7 +271,7 @@ $err[68] = "Already Exists\n"; $err[69] = "No Objectclass Mods\n"; $err[70] = "Results Too Large\n"; $err[71] = "Effect Multiple DSA's\n"; -$err[80] = "Other :-)\n"; +$err[80] = "Other\n"; $err[81] = "Server Down\n"; $err[82] = "Local Error\n"; $err[83] = "Encoding Error\n"; commit 67c5f1cd1584cd352da7b8db2520c268431d7812 Author: Mark Reynolds <mreynolds(a)redhat.com&gt; Date: Tue Jan 8 12:16:14 2013 -0500 Ticket 539 - logconv.pl should handle microsecond timing Bug Description: logconv.pl can not handle microsecond etimes Fix Description: Update the regular expressions to handle the decimal point "." Also did a little code cleanup. https://fedorahosted.org/389/ticket/539 Reviewed by: nhosoi(Thanks!) (cherry picked from commit 8965dda8d4023ab36f0a404a874c657d78631e9d) diff --git a/ldap/admin/src/logconv.pl b/ldap/admin/src/logconv.pl index 94ba036..f5342af 100755 --- a/ldap/admin/src/logconv.pl +++ b/ldap/admin/src/logconv.pl @@ -948,36 +948,31 @@ if ($filter_count > 0){ # # ######################################### - -# -# print most often etimes -# - if ($usage =~ /t/i || $verb eq "yes"){ -print "\n\n----- Top $sizeCount Most Frequent etimes -----\n\n"; -$eloop = 0; -foreach $et (sort { $etime{$b} <=> $etime{$a} } keys %etime) { - if ($eloop == $sizeCount) { last; } - if ($retime ne "2"){ - $first = $et; - $retime = "2"; + # + # print most often etimes + # + print "\n\n----- Top $sizeCount Most Frequent etimes -----\n\n"; + $eloop = 0; + foreach $et (sort { $etime{$b} <=> $etime{$a} } keys %etime) { + if ($eloop == $sizeCount) { last; } + if ($retime ne "2"){ + $first = $et; + $retime = "2"; + } + printf "%-8s %-12s\n", $etime{ $et }, "etime=$et"; + $eloop++; } - printf "%-8s %-12s\n", $etime{ $et }, "etime=$et"; - $eloop++; -} - -# -# print longest etimes -# - -print "\n\n----- Top $sizeCount Longest etimes -----\n\n"; -$eloop = 0; -foreach $et (sort { $b <=> $a } (keys %etime)) { - if ($eloop == $sizeCount) { last; } - printf "%-12s %-10s\n","etime=$et",$etime{ $et }; - $eloop++; -} - + # + # print longest etimes + # + print "\n\n----- Top $sizeCount Longest etimes -----\n\n"; + $eloop = 0; + foreach $et (sort { $b <=> $a } (keys %etime)) { + if ($eloop == $sizeCount) { last; } + printf "%-12s %-10s\n","etime=$et",$etime{ $et }; + $eloop++; + } } ####################################### @@ -1651,7 +1646,7 @@ if (m/ notes=U/){ } if ($usage =~ /u/ || $verb eq "yes"){ if ($v eq "0" ){ - if ($_ =~ /etime= *([0-9]+)/i ) { + if ($_ =~ /etime= *([0-9.]+)/i ) { $notesEtime[$vet]=$1; $vet++; } @@ -1931,7 +1926,7 @@ if ($_ =~ /err= *([0-9]+)/i){ if ($1 ne "0"){ $errorck++;} else { $errorsucc++;} } -if ($_ =~ /etime= *([0-9]+)/i ) { $etime{$1}++;} +if ($_ =~ /etime= *([0-9.]+)/ ) { $etime{$1}++;} if ($_ =~ / tag=101 nentries= *([0-9]+)/i ) {$nentries{$1}++} if ($_ =~ / tag=111 nentries= *([0-9]+)/i ) {$nentries{$1}++} if ($_ =~ / tag=100 nentries= *([0-9]+)/i ) {$nentries{$1}++}