1: #!/usr/bin/perl
2: $|=1;
3: # The LearningOnline Network with CAPA
4: # Cluster Status
5: #
6: # $Id: clusterstatus.pl,v 1.15 2003/08/05 12:52:23 www Exp $
7:
8: use lib '/home/httpd/lib/perl/';
9: use LONCAPA::Configuration;
10:
11: use LWP::UserAgent();
12: use HTTP::Headers;
13: use IO::File;
14:
15: my %host=();
16: my $oneday=60*60*24;
17:
18: my %connectionstatus=();
19: my %perlvar=();
20:
21: my $mode;
22: my $concount=0;
23:
24: sub select_form {
25: my ($def,$name,%hash) = @_;
26: my $selectform = "<select name=\"$name\" size=\"1\">\n";
27: foreach (sort keys %hash) {
28: $selectform.="<option value=\"$_\" ".
29: ($_ eq $def ? 'selected' : '').
30: ">".$hash{$_}."</option>\n";
31: }
32: $selectform.="</select>";
33: return $selectform;
34: }
35:
36:
37: sub key {
38: my ($local,$url)=@_;
39: my $key=$local.'_'.$url;
40: $key=~s/\W/\_/gs;
41: return $key;
42: }
43:
44: sub hidden {
45: my ($name,$value)=@_;
46: print "\n<input type='hidden' name='$name' value='$value' />";
47: }
48:
49: sub request {
50: my ($local,$url,$cachetime)=@_;
51: $cachetime*=(0.5+rand);
52: my $key=&key($local,$url);
53: my $reply='';
54: if ($FORM{$key.'_time'}) {
55: if ((time-$FORM{$key.'_time'})<$cachetime) {
56: $reply=$FORM{$key};
57: &hidden($key.'_time',$FORM{$key.'_time'});
58: &hidden($key.'_fromcache',1);
59: }
60: }
61: unless ($reply) {
62: unless ($hostname{$local}) {
63: $reply='local_unknown';
64: } else {
65:
66: my $ua=new LWP::UserAgent(timeout => 10);
67:
68: my $request=new HTTP::Request('GET',
69: "http://".$hostname{$local}.$url);
70: $request->authorization_basic('lonadm','litelite');
71:
72: my $response=$ua->request($request);
73:
74: unless ($response->is_success) {
75: $reply='local_error';
76: } else {
77: $reply=$response->content;
78: chomp($reply);
79: }
80: }
81: &hidden($key.'_time',time);
82: }
83: &hidden($key,$reply);
84: return $reply;
85: }
86:
87: # ============================================= Are local and remote connected?
88: sub connected {
89: my ($local,$remote)=@_;
90: $local=~s/\W//g;
91: $remote=~s/\W//g;
92:
93: unless ($hostname{$remote}) { return 'remote_unknown'; }
94: my $url='/cgi-bin/ping.pl?'.$remote;
95: #
96: # Slowly phase this in: if not cached, only do 5 percent of the cases,
97: # but always do the first five.
98: #
99: unless ($FORM{&key($local,$url)}) {
100: unless (($concount<=5) || (rand>0.95)) {
101: return 'not_yet';
102: } else {
103: $concount++;
104: }
105: }
106: #
107: # Actually do the query
108: #
109: &statuslist($local,'connecting '.$remote);
110: my $reply=&request($local,$url,3600);
111: $reply=(split("\n",$reply))[0];
112: $reply=~s/\W//g;
113: if ($reply ne $remote) { return $reply; }
114: return 'ok';
115: }
116: # ============================================================ Get a reply hash
117:
118: sub replyhash {
119: my %returnhash=();
120: foreach (split(/\&/,&request(@_))) {
121: my ($name,$value)=split(/\=/,$_);
122: if ($name) {
123: unless ($value) { $value=''; }
124: $returnhash{$name}=$value;
125: }
126: }
127: return %returnhash;
128: }
129:
130: # ================================================================ Link to host
131:
132: sub otherwindow {
133: my ($local,$url,$label)=@_;
134: return
135: " <a href='http://$hostname{$local}$url' target='newwin$local'>$label</a> ";
136: }
137:
138: sub login {
139: my $local=shift;
140: print &otherwindow($local,'/adm/login?domain='.$perlvar{'lonDefDomain'},
141: 'Login');
142: }
143:
144: sub runloncron {
145: my $local=shift;
146: print &otherwindow($local,'/cgi-bin/loncron.pl','Run loncron');
147: }
148:
149: sub loncron {
150: my $local=shift;
151: print &otherwindow($local,'/lon-status','loncron');
152: }
153:
154: sub lonc {
155: my $local=shift;
156: print &otherwindow($local,'/lon-status/loncstatus.txt','lonc');
157: }
158:
159: sub lond {
160: my $local=shift;
161: print &otherwindow($local,'/lon-status/londstatus.txt','lond');
162: }
163:
164: sub users {
165: my $local=shift;
166: print &otherwindow($local,'/cgi-bin/userstatus.pl','Users');
167: }
168:
169: sub versions {
170: my $local=shift;
171: print &otherwindow($local,'/cgi-bin/lonversions.pl','Versions');
172: }
173:
174: sub server {
175: my $local=shift;
176: print &otherwindow($local,'/server-status','Server Status');
177: }
178:
179: # ========================================================= Produce a green bar
180: sub bar {
181: my $parm=shift;
182: my $number=int($parm+0.5);
183: print "<table><tr><td bgcolor='#225522'><font color='#225522'>";
184: for (my $i=0;$i<$number;$i++) {
185: print "+";
186: }
187: print "</font></table>";
188: }
189:
190: # ========================================================== Show server status
191:
192: sub serverstatus {
193: my ($local,$trouble)=@_;
194: print (<<ENDHEADER);
195: <a name="$local" />
196: <table width="100%" bgcolor="#225522" cellspacing="2" cellpadding="2" border="0">
197: <tr><td bgcolor="#BBDDBB"><font color="#225522" face="arial"><b>
198: $local $hostdom{$local}</b> <tt>($hostname{$local}; $hostrole{$local})</tt>
199: <br />$domaindescription{$hostdom{$local}}
200: </font></th></tr><tr><td bgcolor="#DDDDBB"><font color="#225522">
201: ENDHEADER
202: &login($local);&server($local);&users($local);&versions($local);
203: &loncron($local);&lond($local);&lonc($local);&runloncron($local);
204: print "</font></td></tr>";
205: if ($trouble) {
206: print ("<tr><td bgcolor='#DDBBBB'><font color='#552222' size='+2'>$trouble</font></td></tr>");
207: }
208: print "<tr><td bgcolor='#BBBBBB'>";
209: # version
210: if ($host{$local.'_version'}) {
211: print "<br />Version: ".$host{$local.'_version'}
212: }
213: # load
214: if (($host{$local.'_load_doomed'}>0.5) || ($mode eq 'load_doomed')) {
215: print "<br />Load: ".$host{$local.'_load'}
216: }
217: # users
218: if (($host{$local.'_users_doomed'}>10) || ($mode eq 'users_doomed')) {
219: print "<br />Active Users: ".$host{$local.'_users'}
220: }
221:
222: # checkrpms
223: if ($host{$local.'_checkrpms'}) {
224: print "<br />RPMs: ".$host{$local.'_checkrpms'}
225: }
226: # mysql
227: if ($host{$local.'_mysql'}) {
228: print "<br />MySQL Database: ".$host{$local.'_mysql'}
229: }
230: # connections
231: if ($host{$local.'_notconnected'}) {
232: print "<br />Not connected: ";
233: foreach (split(/ /,$host{$local.'_notconnected'})) {
234: if ($_) {
235: print " <a href='#$_'>$_</a>";
236: }
237: }
238: }
239: # errors
240: if ($host{$local.'_errors'}) {
241: print "<br />loncron errors: ".$host{$local.'_errors'};
242: }
243: print "</td></tr></table><br />";
244: }
245:
246: # =========================================================== Doomedness sorted
247:
248: sub doomedness {
249: my $crit=shift;
250: my %alldoomed=();
251: my @allhosts=();
252: foreach (keys %host) {
253: if ($_=~/^(\w+)\_$crit$/) {
254: if ($host{$_}) {
255: push (@allhosts,$1);
256: $alldoomed{$1}=$host{$_};
257: }
258: }
259: }
260: return sort { $alldoomed{$b} <=> $alldoomed{$a} } @allhosts;
261: }
262:
263: # ====================================================================== Status
264: sub statuslist {
265: my ($local,$what)=@_;
266: print
267: "<script>document.prgstat.progress.value='Testing $local ($hostname{$local}): $what';</script>\n";
268: }
269:
270: #
271: # Main program
272: #
273: # ========================================================= Get form parameters
274: my $buffer;
275:
276: read(STDIN, $buffer, $ENV{'CONTENT_LENGTH'});
277: my @pairs=split(/&/,$buffer);
278: my $pair; my $name; my $value;
279: undef %FORM;
280: %FORM=();
281: foreach $pair (@pairs) {
282: ($name,$value) = split(/=/,$pair);
283: $value =~ tr/+/ /;
284: $value =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C",hex($1))/eg;
285: $FORM{$name}=$value;
286: }
287:
288: $buffer=$ENV{'QUERY_STRING'};
289: @pairs=split(/&/,$buffer);
290: foreach $pair (@pairs) {
291: ($name,$value) = split(/=/,$pair);
292: $value =~ tr/+/ /;
293: $value =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C",hex($1))/eg;
294: $FORM{$name}=$value;
295: }
296:
297: # ====================================================== Determine refresh rate
298:
299: my $refresh=(($FORM{'refresh'}=~/^\d+$/)?$FORM{'refresh'}:120);
300: if ($refresh<30) { $refresh=30; }
301: my $starttime=time;
302:
303: # ============================================================== Determine mode
304:
305: my %modes=('trouble' => 'Trouble',
306: 'users_doomed' => 'Doomed: Users',
307: 'loncron_doomed' => 'Doomed: General (loncron)',
308: 'mysql_doomed' => 'Doomed: Database (mysql)',
309: 'notconnected_doomed' => 'Doomed: Connections',
310: 'checkrpms_doomed' => 'Doomed: RPMs',
311: 'load_doomed' => 'Doomed: Load',
312: 'unresponsive_doomed' => 'Doomed: Status could not be determined',
313: 'users' => 'User Report',
314: 'load' => 'Load Report',
315: 'connections' => 'Connections Matrix');
316:
317: $mode=$FORM{'mode'};
318: unless ($modes{$mode}) { $mode='trouble'; }
319: # ================================================================ Send Headers
320: print "Content-type: text/html\n\n".
321: "<html><body bgcolor='#FFFFFF'>\n";
322: # -------------------- Read loncapa.conf (and by default, loncapa_apache.conf).
323: my $perlvarref=LONCAPA::Configuration::read_conf('loncapa.conf');
324: %perlvar=%{$perlvarref};
325: undef $perlvarref; # remove since sensitive and not needed
326: delete $perlvar{'lonReceipt'}; # remove since sensitive and not needed
327: delete $perlvar{'lonSqlAccess'}; # remove since sensitive and not needed
328:
329: # ------------------------------------------------------------- Read hosts file
330: {
331: my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab");
332:
333: $total=0;
334: while (my $configline=<$config>) {
335: $configline=~s/#.*$//;
336: unless ($configline=~/\w/) { next; }
337: my ($id,$domain,$role,$name,$ip)=split(/:/,$configline);
338: $hostname{$id}=$name;
339: $hostdom{$id}=$domain;
340: $hostrole{$id}=$role;
341: $hostip{$id}=$ip;
342: $total++;
343: if (($role eq 'library') && ($id ne $perlvar{'lonHostID'})) {
344: $libserv{$id}=$name;
345: }
346: }
347: }
348: # ------------------------------------------------------------ Read domain file
349: {
350: my $fh=IO::File->new($perlvar{'lonTabDir'}.'/domain.tab');
351: %domaindescription = ();
352: %domain_auth_def = ();
353: %domain_auth_arg_def = ();
354: if ($fh) {
355: while (<$fh>) {
356: next if (/^(\#|\s*$)/);
357: chomp;
358: my ($domain, $domain_description, $def_auth, $def_auth_arg)
359: = split(/:/,$_,4);
360: $domain_auth_def{$domain}=$def_auth;
361: $domain_auth_arg_def{$domain}=$def_auth_arg;
362: $domaindescription{$domain}=$domain_description;
363: }
364: }
365: }
366:
367: print "<img src='/adm/lonIcons/lonlogos.gif' align='right' /><h1>LON-CAPA Cluster Status ".localtime()."</h1>";
368: print "<form name='prgstat'>\n".
369: "<input type='text' name='progress' value='Starting ...' size='100' /><br />".
370: "</form>\n";;
371: print "<form name='status' method='post'>\n";
372: print 'Choose next report: '.&select_form($mode,'mode',%modes).'<hr />';
373: &hidden('refresh',$refresh);
374:
375: # ==================================================== Main Loop over all Hosts
376:
377: my $maxusers=0;
378: my $maxload=0;
379: my $totalusers=0;
380:
381: foreach $local (sort keys %hostname) {
382: $host{$local.'_unresponsive_doomed'}=0;
383: # -- Check general status
384: &statuslist($local,'General');
385: my %loncron=&replyhash($local,'/lon-status/loncron_simple.txt',1200);
386: if (defined($loncron{'local_error'})) {
387: $host{$local.'_loncron'}='Could not determine.';
388: $host{$local.'_unresponsive_doomed'}++;
389: } else {
390: if ((time-$loncron{'time'})>$oneday) {
391: $host{$local.'_loncron'}='Stale.';
392: $host{$local.'_unresponsive_doomed'}++;
393: } else {
394: $host{$local.'_loncron_doomed'}=$loncron{'notices'}
395: +4*$loncron{'warnings'}
396: +100*$loncron{'errors'};
397: $host{$local.'_errors'}=$loncron{'errors'};
398: }
399: }
400: # -- Check version
401: &statuslist($local,'Version');
402: my $version=&request($local,'/lon-status/version.txt',7200);
403: if ($version eq 'local_error') {
404: $host{$local.'_version'}='Could not determine.';
405: $host{$local.'_unresponsive_doomed'}++;
406: } else {
407: $host{$local.'_version'}=$version;
408: }
409: # -- Check user status
410: &statuslist($local,'Users');
411: my %userstatus=&replyhash($local,'/cgi-bin/userstatus.pl?simple',600);
412: if (defined($userstatus{'local_error'})) {
413: $host{$local.'_userstatus'}='Could not determine.';
414: $host{$local.'_unresponsive_doomed'}++;
415: } else {
416: $host{$local.'_users_doomed'}=$userstatus{'Active'};
417: $host{$local.'_users'}=$userstatus{'Active'};
418: unless ($host{$local.'_users'}) { $host{$local.'_users'}=0; }
419: if ($host{$local.'_users'}>$maxusers) {
420: $maxusers=$host{$local.'_users'};
421: }
422: $totalusers+=$host{$local.'_users'};
423: my ($sload,$mload,$lload)=split(/ /,$userstatus{'loadavg'});
424: $host{$local.'_load_doomed'}=$mload;
425: if ($mload>$maxload) {
426: $maxload=$mload;
427: }
428: $host{$local.'_load'}=$userstatus{'loadavg'};
429: }
430: # -- Check mysql status
431: &statuslist($local,'Database');
432: my %mysql=&replyhash($local,'/lon-status/mysql.txt',3600);
433: if (defined($mysql{'local_error'})) {
434: $host{$local.'_mysql'}='Could not determine.';
435: $host{$local.'_unresponsive_doomed'}++;
436: } else {
437: if ((time-$mysql{'time'})>(7*$oneday)) {
438: if ($hostrole{$local} eq 'library') {
439: $host{$local.'_mysql'}='Stale.';
440: $host{$local.'_mysql_doomed'}=1;
441: }
442: if ($mysql{'mysql'} eq 'defunct') {
443: $host{$local.'_mysql'}='Defunct (maybe stale).';
444: $host{$local.'_mysql_doomed'}=2;
445: }
446: } elsif ($mysql{'mysql'} eq 'defunct') {
447: $host{$local.'_mysql'}='Defunct.';
448: $host{$local.'_mysql_doomed'}=3;
449: }
450: }
451: # -- Check rpm status
452: &statuslist($local,'RPMs');
453: my %checkrpms=&replyhash($local,'/lon-status/checkrpms.txt',7200);
454: if (defined($checkrpms{'local_error'})) {
455: $host{$local.'_checkrpms'}='Could not determine.';
456: $host{$local.'_unresponsive_doomed'}++;
457: } else {
458: if ((time-$checkrpms{'time'})>(4*$oneday)) {
459: $host{$local.'_checkrpms'}='Stale.';
460: $host{$local.'_checkrpms_doomed'}=50;
461: $host{$local.'_unresponsive_doomed'}++;
462: } elsif ($checkrpms{'status'} eq 'fail') {
463: $host{$local.'_checkrpms'}='Could not checked RPMs.';
464: $host{$local.'_checkrpms_doomed'}=100;
465: } elsif ($checkrpms{'rpmcount'}) {
466: $host{$local.'_checkrpms'}='Outdated RPMs: '.
467: $checkrpms{'rpmcount'};
468: $host{$local.'_checkrpms_doomed'}=$checkrpms{'rpmcount'};
469: }
470: }
471: # -- Check connections
472: &statuslist($local,'Connections');
473: $host{$local.'_notconnected'}='';
474: $host{$local.'_notconnected_doomed'}=0;
475: foreach $remote (sort keys %hostname) {
476: my $status=&connected($local,$remote);
477: $connectionstatus{$local.'_TO_'.$remote}=$status;
478: unless (($status eq 'ok') || ($status eq 'not_yet')) {
479: $host{$local.'_notconnected'}.=' '.$remote;
480: $host{$local.'_notconnected_doomed'}++;
481: }
482: }
483: # =============================================================== End Mail Loop
484: }
485: &statuslist('Done.');
486: # ====================================================================== Output
487: if ($mode=~/\_doomed$/) {
488: # Output by doomedness
489: foreach (&doomedness($mode)) {
490: &serverstatus($_);
491: }
492: } elsif ($mode eq 'connections') {
493: print
494: "<table cellspacing='3' cellpadding='3' border='0' bgcolor='#225522'>".
495: "<tr><td bgcolor='#225522'> </td>";
496: foreach my $remote (sort keys %hostname) {
497: print '<th bgcolor="#DDDDBB">'.$remote.'</th>';
498: }
499: print "</tr>\n";
500: # connection matrix
501: foreach my $local (sort keys %hostname) {
502: print '<tr><th bgcolor="#DDDDBB">'.$local.'</th>';
503: foreach my $remote (sort keys %hostname) {
504: if ($connectionstatus{$local.'_TO_'.$remote} eq 'not_yet') {
505: my $cellcolor='#FFFFFF';
506: if ($local eq $remote) { $cellcolor='#DDDDDD'; }
507: print '<td bgcolor="'.$cellcolor.'"><font color="#555522" size="-2">not yet tested</font></td>';
508: } elsif ($connectionstatus{$local.'_TO_'.$remote} eq 'ok') {
509: my $cellcolor='#BBDDBB';
510: if ($local eq $remote) { $cellcolor='#99DD99'; }
511: print
512: '<td bgcolor="'.$cellcolor.'"><font color="#225522" face="arial"><b>ok</b></td>';
513: } else {
514: my $cellcolor='#DDBBBB';
515: if ($connectionstatus{$local.'_TO_'.$remote} eq 'local_error') {
516: if ($local eq $remote) {
517: $cellcolor='#DD88AA';
518: } else {
519: $cellcolor='#DDAACC';
520: }
521: } else {
522: if ($local eq $remote) { $cellcolor='#DD9999'; }
523: }
524: print
525: '<td bgcolor="'.$cellcolor.'"><font color="#552222" size="-2">'.
526: $connectionstatus{$local.'_TO_'.$remote}.'<br />';
527: &lonc($local); &lond($remote);
528: print '</td>';
529: }
530: }
531: print "</tr>\n";
532: }
533: print "</table>";
534: } elsif ($mode eq 'users') {
535: # Users
536: if ($maxusers) {
537: my $factor=50/$maxusers;
538: print "<h3>Total active user(s): $totalusers</h3>".
539: "<table cellspacing='3' cellpadding='3' border='0' bgcolor='#225522'>";
540:
541: foreach $local (sort keys %hostname) {
542: if (defined($host{$local.'_users'})) {
543: print
544: '<tr><th bgcolor="#BBDDBB"><font face="arial" color="#225522" size="+1">'.$local.
545: '</font></th><td bgcolor="#DDDDBB">';
546: &users($local);
547: print
548: '</td><td bgcolor="#DDDDBB"><font face="arial" color="#225522">'.
549: $host{$local.'_users'}.'</font></td><td bgcolor="#DDDDBB"';
550: &bar($factor*$host{$local.'_users'});
551: print "</td></tr>\n";
552: }
553: }
554: print "</table>";
555: } else {
556: print "No active users logged in.";
557: }
558: } elsif ($mode eq 'load') {
559: # Load
560: if ($maxload) {
561: my $factor=50/$maxload;
562: print
563: "<table cellspacing='3' cellpadding='3' border='0' bgcolor='#225522'>";
564: foreach $local (sort keys %hostname) {
565: if (defined($host{$local.'_load_doomed'})) {
566: print
567: '<tr><th bgcolor="#BBDDBB"><font face="arial" color="#225522" size="+1">'.
568: $local.
569: '</font></th><td bgcolor="#DDDDBB">';
570: &server($local);
571: print
572: '</td><td bgcolor="#DDDDBB"><font face="arial" color="#225522">'.
573: $host{$local.'_load_doomed'}.'</font></td><td bgcolor="#DDDDBB"';
574: &bar($factor*$host{$local.'_load_doomed'});
575: print "</td></tr>\n";
576: }
577: }
578: print "</table>";
579: } else {
580: print "No workload.";
581: }
582: } elsif ($mode eq 'trouble') {
583: my $count=0;
584: foreach $local (sort keys %hostname) {
585: my $trouble='';
586: if ($host{$local.'_unresponsive_doomed'}>3) {
587: $trouble='Does not respond to several queries.<br />';
588: }
589: if ($host{$local.'_errors'}) {
590: $trouble='Has loncron errors.<br />';
591: } elsif ($host{$local.'_loncron_doomed'}>600) {
592: $trouble='High loncron count.<br />';
593: }
594: if ($host{$local.'_load_doomed'}>5) {
595: $trouble='High load.<br />';
596: }
597: if ($host{$local.'_users_doomed'}>200) {
598: $trouble='High user volume.<br />';
599: }
600: if ($host{$local.'_mysql_doomed'}>1) {
601: $trouble='MySQL database apparently offline.<br />';
602: }
603: if ($host{$local.'_checkrpms_doomed'}>100) {
604: $trouble='RPMs outdated.<br />';
605: }
606: if ($trouble) { $count++; &serverstatus($local,$trouble); }
607: }
608: unless ($count) { print "No mayor trouble."; }
609: }
610: # ============================================================== Close, refresh
611: print "</form><script>";
612: $runtime=time-$starttime;
613: if (($refresh-$runtime)<30) {
614: print "setTimeout('document.status.submit()',30000);\n".
615: "document.prgstat.progress.value='Will automatically refresh.'";
616: } else {
617: $refreshtime=int(1000*($refresh-$runtime));
618: print "setTimeout('document.status.submit()',$refreshtime);\n".
619: "document.prgstat.progress.value='Will automatically refresh ($refresh secs refresh cycle)'";
620: }
621: print "</script></body></html>";
622: exit 0;
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to clusterstatus.pl CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [LON-CAPA] / loncom / cgi