version 1.10, 2003/07/31 16:07:47
|
version 1.15, 2003/08/05 12:52:23
|
Line 19 my %connectionstatus=();
|
Line 19 my %connectionstatus=();
|
my %perlvar=(); |
my %perlvar=(); |
|
|
my $mode; |
my $mode; |
|
my $concount=0; |
|
|
sub select_form { |
sub select_form { |
my ($def,$name,%hash) = @_; |
my ($def,$name,%hash) = @_; |
Line 47 sub hidden {
|
Line 48 sub hidden {
|
|
|
sub request { |
sub request { |
my ($local,$url,$cachetime)=@_; |
my ($local,$url,$cachetime)=@_; |
|
$cachetime*=(0.5+rand); |
my $key=&key($local,$url); |
my $key=&key($local,$url); |
my $reply=''; |
my $reply=''; |
if ($FORM{$key.'_time'}) { |
if ($FORM{$key.'_time'}) { |
Line 61 sub request {
|
Line 63 sub request {
|
$reply='local_unknown'; |
$reply='local_unknown'; |
} else { |
} else { |
|
|
my $ua=new LWP::UserAgent(timeout => 20); |
my $ua=new LWP::UserAgent(timeout => 10); |
|
|
my $request=new HTTP::Request('GET', |
my $request=new HTTP::Request('GET', |
"http://".$hostname{$local}.$url); |
"http://".$hostname{$local}.$url); |
Line 91 sub connected {
|
Line 93 sub connected {
|
unless ($hostname{$remote}) { return 'remote_unknown'; } |
unless ($hostname{$remote}) { return 'remote_unknown'; } |
my $url='/cgi-bin/ping.pl?'.$remote; |
my $url='/cgi-bin/ping.pl?'.$remote; |
# |
# |
# Slowly phase this in: if not cached, only do 10 percent of the cases |
# Slowly phase this in: if not cached, only do 5 percent of the cases, |
|
# but always do the first five. |
# |
# |
unless ($FORM{&key($local,$url)}) { |
unless ($FORM{&key($local,$url)}) { |
unless (rand>0.9) { return 'not_yet'; } |
unless (($concount<=5) || (rand>0.95)) { |
|
return 'not_yet'; |
|
} else { |
|
$concount++; |
|
} |
} |
} |
# |
# |
# Actually do the query |
# Actually do the query |
Line 169 sub server {
|
Line 176 sub server {
|
print &otherwindow($local,'/server-status','Server Status'); |
print &otherwindow($local,'/server-status','Server Status'); |
} |
} |
|
|
|
# ========================================================= Produce a green bar |
|
sub bar { |
|
my $parm=shift; |
|
my $number=int($parm+0.5); |
|
print "<table><tr><td bgcolor='#225522'><font color='#225522'>"; |
|
for (my $i=0;$i<$number;$i++) { |
|
print "+"; |
|
} |
|
print "</font></table>"; |
|
} |
|
|
# ========================================================== Show server status |
# ========================================================== Show server status |
|
|
sub serverstatus { |
sub serverstatus { |
my $local=shift; |
my ($local,$trouble)=@_; |
print (<<ENDHEADER); |
print (<<ENDHEADER); |
|
<a name="$local" /> |
<table width="100%" bgcolor="#225522" cellspacing="2" cellpadding="2" border="0"> |
<table width="100%" bgcolor="#225522" cellspacing="2" cellpadding="2" border="0"> |
<tr><td bgcolor="#BBDDBB"><font color="#225522" face="arial"><b> |
<tr><td bgcolor="#BBDDBB"><font color="#225522" face="arial"><b> |
$local $hostdom{$local}</b> <tt>($hostname{$local}; $hostrole{$local})</tt> |
$local $hostdom{$local}</b> <tt>($hostname{$local}; $hostrole{$local})</tt> |
Line 182 $local $hostdom{$local}</b> <tt>($hostna
|
Line 201 $local $hostdom{$local}</b> <tt>($hostna
|
ENDHEADER |
ENDHEADER |
&login($local);&server($local);&users($local);&versions($local); |
&login($local);&server($local);&users($local);&versions($local); |
&loncron($local);&lond($local);&lonc($local);&runloncron($local); |
&loncron($local);&lond($local);&lonc($local);&runloncron($local); |
print "</font></td></tr><tr><td bgcolor='#BBBBBB'>"; |
print "</font></td></tr>"; |
|
if ($trouble) { |
|
print ("<tr><td bgcolor='#DDBBBB'><font color='#552222' size='+2'>$trouble</font></td></tr>"); |
|
} |
|
print "<tr><td bgcolor='#BBBBBB'>"; |
|
# version |
|
if ($host{$local.'_version'}) { |
|
print "<br />Version: ".$host{$local.'_version'} |
|
} |
# load |
# load |
if (($host{$local.'_load_doomed'}>0.5) || ($mode eq 'load_doomed')) { |
if (($host{$local.'_load_doomed'}>0.5) || ($mode eq 'load_doomed')) { |
print "<br />Load: ".$host{$local.'_load'} |
print "<br />Load: ".$host{$local.'_load'} |
Line 200 ENDHEADER
|
Line 227 ENDHEADER
|
if ($host{$local.'_mysql'}) { |
if ($host{$local.'_mysql'}) { |
print "<br />MySQL Database: ".$host{$local.'_mysql'} |
print "<br />MySQL Database: ".$host{$local.'_mysql'} |
} |
} |
|
# connections |
|
if ($host{$local.'_notconnected'}) { |
|
print "<br />Not connected: "; |
|
foreach (split(/ /,$host{$local.'_notconnected'})) { |
|
if ($_) { |
|
print " <a href='#$_'>$_</a>"; |
|
} |
|
} |
|
} |
|
# errors |
|
if ($host{$local.'_errors'}) { |
|
print "<br />loncron errors: ".$host{$local.'_errors'}; |
|
} |
print "</td></tr></table><br />"; |
print "</td></tr></table><br />"; |
} |
} |
|
|
Line 256 foreach $pair (@pairs) {
|
Line 296 foreach $pair (@pairs) {
|
|
|
# ====================================================== Determine refresh rate |
# ====================================================== Determine refresh rate |
|
|
my $refresh=(($FORM{'refresh'}=~/^\d+$/)?$FORM{'refresh'}:60); |
my $refresh=(($FORM{'refresh'}=~/^\d+$/)?$FORM{'refresh'}:120); |
if ($refresh<30) { $refresh=30; } |
if ($refresh<30) { $refresh=30; } |
my $starttime=time; |
my $starttime=time; |
|
|
Line 271 my %modes=('trouble' => 'Trouble',
|
Line 311 my %modes=('trouble' => 'Trouble',
|
'load_doomed' => 'Doomed: Load', |
'load_doomed' => 'Doomed: Load', |
'unresponsive_doomed' => 'Doomed: Status could not be determined', |
'unresponsive_doomed' => 'Doomed: Status could not be determined', |
'users' => 'User Report', |
'users' => 'User Report', |
|
'load' => 'Load Report', |
'connections' => 'Connections Matrix'); |
'connections' => 'Connections Matrix'); |
|
|
$mode=$FORM{'mode'}; |
$mode=$FORM{'mode'}; |
Line 323 delete $perlvar{'lonSqlAccess'}; # remov
|
Line 364 delete $perlvar{'lonSqlAccess'}; # remov
|
} |
} |
} |
} |
|
|
|
|
print "<img src='/adm/lonIcons/lonlogos.gif' align='right' /><h1>LON-CAPA Cluster Status ".localtime()."</h1>"; |
print "<img src='/adm/lonIcons/lonlogos.gif' align='right' /><h1>LON-CAPA Cluster Status ".localtime()."</h1>"; |
print "<form name='prgstat'>\n". |
print "<form name='prgstat'>\n". |
"<input type='text' name='progress' value='Starting ...' size='100' /><br />". |
"<input type='text' name='progress' value='Starting ...' size='100' /><br />". |
Line 334 print 'Choose next report: '.&select_for
|
Line 374 print 'Choose next report: '.&select_for
|
|
|
# ==================================================== Main Loop over all Hosts |
# ==================================================== Main Loop over all Hosts |
|
|
|
my $maxusers=0; |
|
my $maxload=0; |
|
my $totalusers=0; |
|
|
foreach $local (sort keys %hostname) { |
foreach $local (sort keys %hostname) { |
$host{$local.'_unresponsive_doomed'}=0; |
$host{$local.'_unresponsive_doomed'}=0; |
# -- Check general status |
# -- Check general status |
Line 347 foreach $local (sort keys %hostname) {
|
Line 391 foreach $local (sort keys %hostname) {
|
$host{$local.'_loncron'}='Stale.'; |
$host{$local.'_loncron'}='Stale.'; |
$host{$local.'_unresponsive_doomed'}++; |
$host{$local.'_unresponsive_doomed'}++; |
} else { |
} else { |
|
$host{$local.'_loncron_doomed'}=$loncron{'notices'} |
|
+4*$loncron{'warnings'} |
|
+100*$loncron{'errors'}; |
|
$host{$local.'_errors'}=$loncron{'errors'}; |
} |
} |
} |
} |
|
# -- Check version |
|
&statuslist($local,'Version'); |
|
my $version=&request($local,'/lon-status/version.txt',7200); |
|
if ($version eq 'local_error') { |
|
$host{$local.'_version'}='Could not determine.'; |
|
$host{$local.'_unresponsive_doomed'}++; |
|
} else { |
|
$host{$local.'_version'}=$version; |
|
} |
# -- Check user status |
# -- Check user status |
&statuslist($local,'Users'); |
&statuslist($local,'Users'); |
my %userstatus=&replyhash($local,'/cgi-bin/userstatus.pl?simple',600); |
my %userstatus=&replyhash($local,'/cgi-bin/userstatus.pl?simple',600); |
Line 358 foreach $local (sort keys %hostname) {
|
Line 415 foreach $local (sort keys %hostname) {
|
} else { |
} else { |
$host{$local.'_users_doomed'}=$userstatus{'Active'}; |
$host{$local.'_users_doomed'}=$userstatus{'Active'}; |
$host{$local.'_users'}=$userstatus{'Active'}; |
$host{$local.'_users'}=$userstatus{'Active'}; |
|
unless ($host{$local.'_users'}) { $host{$local.'_users'}=0; } |
|
if ($host{$local.'_users'}>$maxusers) { |
|
$maxusers=$host{$local.'_users'}; |
|
} |
|
$totalusers+=$host{$local.'_users'}; |
my ($sload,$mload,$lload)=split(/ /,$userstatus{'loadavg'}); |
my ($sload,$mload,$lload)=split(/ /,$userstatus{'loadavg'}); |
$host{$local.'_load_doomed'}=$mload; |
$host{$local.'_load_doomed'}=$mload; |
|
if ($mload>$maxload) { |
|
$maxload=$mload; |
|
} |
$host{$local.'_load'}=$userstatus{'loadavg'}; |
$host{$local.'_load'}=$userstatus{'loadavg'}; |
} |
} |
# -- Check mysql status |
# -- Check mysql status |
Line 428 foreach $local (sort keys %hostname) {
|
Line 493 foreach $local (sort keys %hostname) {
|
print |
print |
"<table cellspacing='3' cellpadding='3' border='0' bgcolor='#225522'>". |
"<table cellspacing='3' cellpadding='3' border='0' bgcolor='#225522'>". |
"<tr><td bgcolor='#225522'> </td>"; |
"<tr><td bgcolor='#225522'> </td>"; |
foreach (sort keys %hostname) { |
foreach my $remote (sort keys %hostname) { |
my $remote=$_; |
|
print '<th bgcolor="#DDDDBB">'.$remote.'</th>'; |
print '<th bgcolor="#DDDDBB">'.$remote.'</th>'; |
} |
} |
print "</tr>\n"; |
print "</tr>\n"; |
# connection matrix |
# connection matrix |
foreach (sort keys %hostname) { |
foreach my $local (sort keys %hostname) { |
my $local=$_; |
|
print '<tr><th bgcolor="#DDDDBB">'.$local.'</th>'; |
print '<tr><th bgcolor="#DDDDBB">'.$local.'</th>'; |
foreach (sort keys %hostname) { |
foreach my $remote (sort keys %hostname) { |
my $remote=$_; |
|
if ($connectionstatus{$local.'_TO_'.$remote} eq 'not_yet') { |
if ($connectionstatus{$local.'_TO_'.$remote} eq 'not_yet') { |
print '<td bgcolor="#FFFFFF"> </td>'; |
my $cellcolor='#FFFFFF'; |
|
if ($local eq $remote) { $cellcolor='#DDDDDD'; } |
|
print '<td bgcolor="'.$cellcolor.'"><font color="#555522" size="-2">not yet tested</font></td>'; |
} elsif ($connectionstatus{$local.'_TO_'.$remote} eq 'ok') { |
} elsif ($connectionstatus{$local.'_TO_'.$remote} eq 'ok') { |
|
my $cellcolor='#BBDDBB'; |
|
if ($local eq $remote) { $cellcolor='#99DD99'; } |
print |
print |
'<td bgcolor="#BBDDBB"><font color="#225522" face="arial"><b>ok</b></td>'; |
'<td bgcolor="'.$cellcolor.'"><font color="#225522" face="arial"><b>ok</b></td>'; |
} else { |
} else { |
|
my $cellcolor='#DDBBBB'; |
|
if ($connectionstatus{$local.'_TO_'.$remote} eq 'local_error') { |
|
if ($local eq $remote) { |
|
$cellcolor='#DD88AA'; |
|
} else { |
|
$cellcolor='#DDAACC'; |
|
} |
|
} else { |
|
if ($local eq $remote) { $cellcolor='#DD9999'; } |
|
} |
print |
print |
'<td bgcolor="#DDBBBB"><font color="#552222" size="-2">'. |
'<td bgcolor="'.$cellcolor.'"><font color="#552222" size="-2">'. |
$connectionstatus{$local.'_TO_'.$remote}.'<br />'; |
$connectionstatus{$local.'_TO_'.$remote}.'<br />'; |
&lonc($local); &lond($remote); |
&lonc($local); &lond($remote); |
print '</td>'; |
print '</td>'; |
Line 454 foreach $local (sort keys %hostname) {
|
Line 530 foreach $local (sort keys %hostname) {
|
} |
} |
print "</tr>\n"; |
print "</tr>\n"; |
} |
} |
|
print "</table>"; |
|
} elsif ($mode eq 'users') { |
|
# Users |
|
if ($maxusers) { |
|
my $factor=50/$maxusers; |
|
print "<h3>Total active user(s): $totalusers</h3>". |
|
"<table cellspacing='3' cellpadding='3' border='0' bgcolor='#225522'>"; |
|
|
|
foreach $local (sort keys %hostname) { |
|
if (defined($host{$local.'_users'})) { |
|
print |
|
'<tr><th bgcolor="#BBDDBB"><font face="arial" color="#225522" size="+1">'.$local. |
|
'</font></th><td bgcolor="#DDDDBB">'; |
|
&users($local); |
|
print |
|
'</td><td bgcolor="#DDDDBB"><font face="arial" color="#225522">'. |
|
$host{$local.'_users'}.'</font></td><td bgcolor="#DDDDBB"'; |
|
&bar($factor*$host{$local.'_users'}); |
|
print "</td></tr>\n"; |
|
} |
|
} |
|
print "</table>"; |
|
} else { |
|
print "No active users logged in."; |
|
} |
|
} elsif ($mode eq 'load') { |
|
# Load |
|
if ($maxload) { |
|
my $factor=50/$maxload; |
|
print |
|
"<table cellspacing='3' cellpadding='3' border='0' bgcolor='#225522'>"; |
|
foreach $local (sort keys %hostname) { |
|
if (defined($host{$local.'_load_doomed'})) { |
|
print |
|
'<tr><th bgcolor="#BBDDBB"><font face="arial" color="#225522" size="+1">'. |
|
$local. |
|
'</font></th><td bgcolor="#DDDDBB">'; |
|
&server($local); |
|
print |
|
'</td><td bgcolor="#DDDDBB"><font face="arial" color="#225522">'. |
|
$host{$local.'_load_doomed'}.'</font></td><td bgcolor="#DDDDBB"'; |
|
&bar($factor*$host{$local.'_load_doomed'}); |
|
print "</td></tr>\n"; |
|
} |
|
} |
|
print "</table>"; |
|
} else { |
|
print "No workload."; |
|
} |
|
} elsif ($mode eq 'trouble') { |
|
my $count=0; |
|
foreach $local (sort keys %hostname) { |
|
my $trouble=''; |
|
if ($host{$local.'_unresponsive_doomed'}>3) { |
|
$trouble='Does not respond to several queries.<br />'; |
|
} |
|
if ($host{$local.'_errors'}) { |
|
$trouble='Has loncron errors.<br />'; |
|
} elsif ($host{$local.'_loncron_doomed'}>600) { |
|
$trouble='High loncron count.<br />'; |
|
} |
|
if ($host{$local.'_load_doomed'}>5) { |
|
$trouble='High load.<br />'; |
|
} |
|
if ($host{$local.'_users_doomed'}>200) { |
|
$trouble='High user volume.<br />'; |
|
} |
|
if ($host{$local.'_mysql_doomed'}>1) { |
|
$trouble='MySQL database apparently offline.<br />'; |
|
} |
|
if ($host{$local.'_checkrpms_doomed'}>100) { |
|
$trouble='RPMs outdated.<br />'; |
|
} |
|
if ($trouble) { $count++; &serverstatus($local,$trouble); } |
|
} |
|
unless ($count) { print "No mayor trouble."; } |
} |
} |
print "</table>"; |
|
# ============================================================== Close, refresh |
# ============================================================== Close, refresh |
print "</form><script>"; |
print "</form><script>"; |
$runtime=time-$starttime; |
$runtime=time-$starttime; |
if ($runtime>=$refresh) { |
if (($refresh-$runtime)<30) { |
print 'document.status.submit();'; |
print "setTimeout('document.status.submit()',30000);\n". |
|
"document.prgstat.progress.value='Will automatically refresh.'"; |
} else { |
} else { |
$refreshtime=int(1000*($refresh-$runtime)); |
$refreshtime=int(1000*($refresh-$runtime)); |
print "setTimeout('document.status.submit()',$refreshtime);"; |
print "setTimeout('document.status.submit()',$refreshtime);\n". |
|
"document.prgstat.progress.value='Will automatically refresh ($refresh secs refresh cycle)'"; |
} |
} |
print "</script></body></html>"; |
print "</script></body></html>"; |
exit 0; |
exit 0; |