--- loncom/loncron 2001/04/21 15:22:37 1.20 +++ loncom/loncron 2003/06/10 22:07:26 1.39 @@ -9,19 +9,18 @@ # # 7/14,7/15,7/19,7/21,7/22,11/18, # 2/8 Gerd Kortemeyer -# 12/6/2000,12/8 Scott Harrison # 12/23 Gerd Kortemeyer -# 1/10/2001, 2/12/, 2/26, 3/15, 04/11, 04/21 Scott Harrison +# YEAR=2001 +# 09/04,09/06,11/26 Gerd Kortemeyer + +$|=1; + +use lib '/home/httpd/lib/perl/'; +use LONCAPA::Configuration; use IO::File; use IO::Socket; -my $qflag=0; -if (@ARGV) { - my $arg=shift @ARGV; - $qflag=1 if $arg eq 'quick'; -} - # -------------------------------------------------- Non-critical communication sub reply { my ($cmd,$server)=@_; @@ -52,19 +51,12 @@ ENDERROUT # ================================================================ Main Program -# ------------------------------------------------------------ Read access.conf -{ - my $config=IO::File->new("/etc/httpd/conf/access.conf"); - - while (my $configline=<$config>) { - if ($configline =~ /PerlSetVar/) { - my ($dummy,$varname,$varvalue)=split(/\s+/,$configline); - $perlvar{$varname}=$varvalue; - } - } - delete $perlvar{'lonReceipt'}; # remove since sensitive and not needed - delete $perlvar{'lonSqlAccess'}; # remove since sensitive and not needed -} +# --------------------------------- Read loncapa_apache.conf and loncapa.conf +my $perlvarref=LONCAPA::Configuration::read_conf('loncapa.conf'); +%perlvar=%{$perlvarref}; +undef $perlvarref; +delete $perlvar{'lonReceipt'}; # remove since sensitive and not needed +delete $perlvar{'lonSqlAccess'}; # remove since sensitive and not needed # --------------------------------------- Make sure that LON-CAPA is configured # I only test for one thing here (lonHostID). This is just a safeguard. @@ -96,14 +88,21 @@ if ($wwwid!=$<) { my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab"); while (my $configline=<$config>) { - my ($id,$domain,$role,$name,$ip)=split(/:/,$configline); - $hostname{$id}=$name; - $hostdom{$id}=$domain; - $hostrole{$id}=$role; - $hostip{$id}=$ip; - if (($role eq 'library') && ($id ne $perlvar{'lonHostID'})) { - $libserv{$id}=$name; - } + my ($id,$domain,$role,$name,$ip,$domdescr)=split(/:/,$configline); + if ($id && $domain && $role && $name && $ip) { + $hostname{$id}=$name; + $hostdom{$id}=$domain; + $hostip{$id}=$ip; + $hostrole{$id}=$role; + if ($domdescr) { $domaindescription{$domain}=$domdescr; } + if (($role eq 'library') && ($id ne $perlvar{'lonHostID'})) { + $libserv{$id}=$name; + } + } else { + if ($configline) { +# &logthis("Skipping hosts.tab line -$configline-"); + } + } } } @@ -151,6 +150,7 @@ print $fh (<lonsql
  • lond
  • lonc +
  • lonhttpd
  • lonnet
  • Connections
  • Delayed Messages @@ -163,17 +163,17 @@ print $fh (< ENDHEADERS -foreach $varname (keys %perlvar) { +foreach $varname (sort(keys(%perlvar))) { print $fh "$varname$perlvar{$varname}\n"; } print $fh "

    Hosts

    "; -foreach $id (keys %hostname) { -print $fh - ""; -print $fh "\n"; +foreach $id (sort(keys(%hostname))) { + print $fh + ""; + print $fh "\n"; } print $fh "
    $id$hostdom{$id}$hostrole{$id}$hostname{$id}$hostip{$id}
    $id$hostdom{$id}$hostrole{$id}$hostname{$id}$hostip{$id}

    Spare Hosts

      "; -foreach $id (keys %spareid) { +foreach $id (sort(keys(%spareid))) { print $fh "
    1. $id\n"; } @@ -209,7 +209,8 @@ while ($line=) { $usage=$parts[4]; $usage=~s/\W//g; if ($usage>90) { - $warnings++; + $warnings++; + $notices++; } elsif ($usage>80) { $warnings++; } elsif ($usage>60) { @@ -219,11 +220,29 @@ while ($line=) { } close (DFH); print $fh ""; + + +print $fh "

      ps

      "; +print $fh "
      ";
+$psproc=0;
+
+open (PSH,"ps -aux|");
+while ($line=) { 
+   print $fh "$line"; 
+   $psproc++;
+}
+close (PSH);
+print $fh "
      "; + +if ($psproc>200) { $notices++; } +if ($psproc>250) { $notices++; } + &errout($fh); # --------------------------------------------------------------- clean out tmp print $fh '

      Temporary Files

      '; $cleaned=0; +$old=0; while ($fname=<$perlvar{'lonDaemons'}/tmp/*>) { my ($dev,$ino,$mode,$nlink, $uid,$gid,$rdev,$size, @@ -232,12 +251,24 @@ while ($fname=<$perlvar{'lonDaemons'}/tm $now=time; $since=$now-$mtime; if ($since>$perlvar{'lonExpire'}) { - $cleaned++; - unlink("$fname"); + $line=''; + if (open(PROBE,$fname)) { + $line=; + close(PROBE); + } + unless ($line=~/^CHECKOUTTOKEN\&/) { + $cleaned++; + unlink("$fname"); + } else { + if ($since>365*$perlvar{'lonExpire'}) { + $cleaned++; + unlink("$fname"); + } else { $old++; } + } } } -print $fh "Cleaned up ".$cleaned." files."; +print $fh "Cleaned up ".$cleaned." files (".$old." old checkout tokens)."; # ------------------------------------------------------------ clean out lonIDs print $fh '

      Session Tokens

      '; @@ -266,13 +297,13 @@ print $fh "

      $active open session(s)

      httpd

      Access Log

      ';
 
-open (DFH,"tail -n40 /etc/httpd/logs/access_log|");
+open (DFH,"tail -n25 /etc/httpd/logs/access_log|");
 while ($line=) { print $fh "$line" };
 close (DFH);
 
 print $fh "

      Error Log

      ";
 
-open (DFH,"tail -n50 /etc/httpd/logs/error_log|");
+open (DFH,"tail -n25 /etc/httpd/logs/error_log|");
 while ($line=) { 
    print $fh "$line";
    if ($line=~/\[error\]/) { $notices++; } 
@@ -283,13 +314,10 @@ print $fh "
      "; # ---------------------------------------------------------------------- lonsql -# -# Do not run for now -# -if ($perlvar{'lonRole'} eq "library") { +my $restartflag=1; print $fh '

      lonsql

      Log

      ';
-    
+    print "lonsql\n";
     if (-e "$perlvar{'lonDaemons'}/logs/lonsql.log"){
 	open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonsql.log|");
 	while ($line=) { 
@@ -303,29 +331,42 @@ if ($perlvar{'lonRole'} eq "library") {
     print $fh "
      "; my $lonsqlfile="$perlvar{'lonDaemons'}/logs/lonsql.pid"; - + + $restartflag=1; + if (-e $lonsqlfile) { my $lfh=IO::File->new("$lonsqlfile"); my $lonsqlpid=<$lfh>; chomp($lonsqlpid); if (kill 0 => $lonsqlpid) { print $fh "

      lonsql at pid $lonsqlpid responding

      "; + $restartflag=0; } else { $errors++; $errors++; print $fh "

      lonsql at pid $lonsqlpid not responding

      "; + $restartflag=1; + print $fh + "

      Decided to clean up stale .pid file and restart lonsql

      "; } - } else { + } + if ($restartflag==1) { $errors++; + print $fh '
      Killall lonsql: '. + system('killall lonsql').' - '; + sleep 2; + print $fh unlink($lonsqlfile).' - '. + system('killall -9 lonsql'). + '
      '; print $fh "

      lonsql not running, trying to start

      "; system( "$perlvar{'lonDaemons'}/lonsql 2>>$perlvar{'lonDaemons'}/logs/lonsql_errors"); - sleep 120 unless $qflag; + sleep 2; if (-e $lonsqlfile) { print $fh "Seems like it started ...

      "; my $lfh=IO::File->new("$lonsqlfile"); my $lonsqlpid=<$lfh>; chomp($lonsqlpid); - sleep 30 unless $qflag; + sleep 2; if (kill 0 => $lonsqlpid) { print $fh "

      lonsql at pid $lonsqlpid responding

      "; } else { @@ -334,7 +375,7 @@ if ($perlvar{'lonRole'} eq "library") { print $fh "Give it one more try ...

      "; system( "$perlvar{'lonDaemons'}/lonsql 2>>$perlvar{'lonDaemons'}/logs/lonsql_errors"); - sleep 120 unless $qflag; + sleep 2; } } else { print $fh "Seems like that did not work!

      "; @@ -360,7 +401,7 @@ if ($perlvar{'lonRole'} eq "library") { $atime,$mtime,$ctime, $blksize,$blocks)=stat($fname); - if ($size>40000) { + if ($size>200000) { print $fh "Rotating logs ...

      "; rename("$fname.2","$fname.3"); rename("$fname.1","$fname.2"); @@ -368,13 +409,13 @@ if ($perlvar{'lonRole'} eq "library") { } &errout($fh); -} # ------------------------------------------------------------------------ lond print $fh '

      lond

      Log

      ';
+print "lond\n";
 
 if (-e "$perlvar{'lonDaemons'}/logs/lond.log"){
-open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lond.log|");
+open (DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/lond.log|");
 while ($line=) { 
    print $fh "$line";
    if ($line=~/INFO/) { $notices++; }
@@ -387,48 +428,40 @@ print $fh "
      "; my $londfile="$perlvar{'lonDaemons'}/logs/lond.pid"; -my $restartflag=1; +$restartflag=1; if (-e $londfile) { my $lfh=IO::File->new("$londfile"); my $londpid=<$lfh>; chomp($londpid); if (kill 0 => $londpid) { - print $fh "

      lond at pid $londpid responding

      "; + print $fh "

      lond at pid $londpid responding, sending USR1

      "; + kill USR1 => $londpid; $restartflag=0; } else { $errors++; print $fh "

      lond at pid $londpid not responding

      "; - # Intelligently handle this. - # Possibility #1: there is no process - # Solution: remove .pid file and restart - if (getpgrp($londpid)==-1) { - unlink($londfile); - $restartflag=1; - } - else { - # Possibility #2: there is a live process that is not responding - # for an unknown reason - # Solution: kill parent and children processes, remove .pid and restart - `killall -9 lond`; - unlink($londfile); - $restartflag=1; - } + $restartflag=1; print $fh - "

      Deciding to clean up stale .pid file and restart lond

      "; + "

      Decided to clean up stale .pid file and restart lond

      "; } } if ($restartflag==1) { $errors++; + print $fh '
      Killall lond: '. + system('killall lond').' - '; + sleep 2; + print $fh unlink($londfile).' - '.system('killall -9 lond'). + '
      '; print $fh "

      lond not running, trying to start

      "; system( "$perlvar{'lonDaemons'}/lond 2>>$perlvar{'lonDaemons'}/logs/lond_errors"); - sleep 120 unless $qflag; + sleep 2; if (-e $londfile) { print $fh "Seems like it started ...

      "; my $lfh=IO::File->new("$londfile"); my $londpid=<$lfh>; chomp($londpid); - sleep 30 unless $qflag; + sleep 2; if (kill 0 => $londpid) { print $fh "

      lond at pid $londpid responding

      "; } else { @@ -437,7 +470,7 @@ if ($restartflag==1) { print $fh "Give it one more try ...

      "; system( "$perlvar{'lonDaemons'}/lond 2>>$perlvar{'lonDaemons'}/logs/lond_errors"); - sleep 120 unless $qflag; + sleep 2; } } else { print $fh "Seems like that did not work!

      "; @@ -474,9 +507,10 @@ if ($size>40000) { # ------------------------------------------------------------------------ lonc print $fh '

      lonc

      Log

      ';
+print "lonc\n";
 
 if (-e "$perlvar{'lonDaemons'}/logs/lonc.log"){
-open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonc.log|");
+open (DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/lonc.log|");
 while ($line=) { 
    print $fh "$line";
    if ($line=~/INFO/) { $notices++; }
@@ -501,37 +535,29 @@ if (-e $loncfile) {
    } else {
       $errors++;
       print $fh "
      lonc at pid $loncpid not responding
      ";
-      # Intelligently handle this.
-      # Possibility #1: there is no process
-      # Solution: remove .pid file and restart
-      if (getpgrp($loncpid)==-1) {
-	  unlink($loncfile);
-	  $restartflag=1;
-      }
-      else {
-      # Possibility #2: there is a live process that is not responding
-      #                 for an unknown reason
       # Solution: kill parent and children processes, remove .pid and restart
-	  `killall -9 lonc`;
-	  unlink($loncfile);
 	  $restartflag=1;
-      }
       print $fh 
-	  "
      Deciding to clean up stale .pid file and restart lonc
      ";
+	  "
      Decided to clean up stale .pid file and restart lonc
      ";
    }
 } 
 if ($restartflag==1) {
    $errors++;
+	  print $fh '
      Killall lonc: '.
+	            system('killall lonc').' - ';
+          sleep 2;
+          print $fh unlink($loncfile).' - '.system('killall -9 lonc').
+                    '
      ';
    print $fh "
      lonc not running, trying to start
      ";
 	system(
  "$perlvar{'lonDaemons'}/lonc 2>>$perlvar{'lonDaemons'}/logs/lonc_errors");
-   sleep 120 unless $qflag;
+   sleep 2;
    if (-e $loncfile) {
        print $fh "Seems like it started ...
      ";
        my $lfh=IO::File->new("$loncfile");
        my $loncpid=<$lfh>;
        chomp($loncpid);
-       sleep 30 unless $qflag;
+       sleep 2;
        if (kill 0 => $loncpid) {
           print $fh "
      lonc at pid $loncpid responding
      ";
        } else {
@@ -540,7 +566,7 @@ if ($restartflag==1) {
           print $fh "Give it one more try ...
      ";
  	  system(
  "$perlvar{'lonDaemons'}/lonc 2>>$perlvar{'lonDaemons'}/logs/lonc_errors");
-          sleep 120 unless $qflag;
+          sleep 2;
        }
    } else {
        print $fh "Seems like that did not work!
      ";
@@ -575,9 +601,106 @@ if ($size>40000) {
 
    
 &errout($fh);
+# -------------------------------------------------------------------- lonhttpd
+
+print $fh '
      lonhttpd
      Log
      ';
+print "lonhttpd\n";
+
+if (-e "$perlvar{'lonDaemons'}/logs/lonhttpd.log"){
+open (DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/lonhttpd.log|");
+while ($line=) { 
+   print $fh "$line";
+   if ($line=~/INFO/) { $notices++; }
+   if ($line=~/WARNING/) { $notices++; }
+   if ($line=~/CRITICAL/) { $warnings++; }
+};
+close (DFH);
+}
+print $fh "
      ";
+
+my $lonhttpdfile="$perlvar{'lonDaemons'}/logs/lonhttpd.pid";
+
+$restartflag=1;
+if (-e $lonhttpdfile) {
+   my $lfh=IO::File->new("$lonhttpdfile");
+   my $lonhttpdpid=<$lfh>;
+   chomp($lonhttpdpid);
+   if (kill 0 => $lonhttpdpid) {
+      print $fh "
      lonhttpd at pid $lonhttpdpid responding
      ";
+      $restartflag=0;
+   } else {
+      $errors++;
+      print $fh "
      lonhttpd at pid $lonhttpdpid not responding
      ";
+      # Solution: kill parent and children processes, remove .pid and restart
+	  $restartflag=1;
+      print $fh 
+	  "
      Decided to clean up stale .pid file and restart lonhttpd
      ";
+   }
+} 
+if ($restartflag==1) {
+   $errors++;
+	  print $fh '
      Killall lonhttpd: '.
+	            system('killall lonhttpd').' - ';
+          sleep 2;
+          print $fh unlink($lonhttpdfile).' - '.system('killall -9 lonhttpd').
+                    '
      ';
+   print $fh "
      lonhttpd not running, trying to start
      ";
+	system(
+ "$perlvar{'lonDaemons'}/lonhttpd 2>>$perlvar{'lonDaemons'}/logs/lonhttpd_errors");
+   sleep 2;
+   if (-e $lonhttpdfile) {
+       print $fh "Seems like it started ...
      ";
+       my $lfh=IO::File->new("$lonhttpdfile");
+       my $lonhttpdpid=<$lfh>;
+       chomp($lonhttpdpid);
+       sleep 2;
+       if (kill 0 => $lonhttpdpid) {
+          print $fh "
      lonhttpd at pid $lonhttpdpid responding
      ";
+       } else {
+          $errors++; $errors++;
+          print $fh "
      lonhttpd at pid $lonhttpdpid not responding
      ";
+          print $fh "Give it one more try ...
      ";
+ 	  system(
+ "$perlvar{'lonDaemons'}/lonhttpd 2>>$perlvar{'lonDaemons'}/logs/lonhttpd_errors");
+          sleep 2;
+       }
+   } else {
+       print $fh "Seems like that did not work!
      ";
+       $errors++;
+   }
+   if (-e "$perlvar{'lonDaemons'}/logs/lonhttpd.log") {
+    print $fh "
      ";
+    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonhttpd.log|");
+    while ($line=) { 
+      print $fh "$line";
+      if ($line=~/WARNING/) { $notices++; }
+      if ($line=~/CRITICAL/) { $notices++; }
+    };
+    close (DFH);
+    print $fh "
      ";
+   }
+}
+
+$fname="$perlvar{'lonDaemons'}/logs/lonhttpd.log";
+
+                          my ($dev,$ino,$mode,$nlink,
+                              $uid,$gid,$rdev,$size,
+                              $atime,$mtime,$ctime,
+                              $blksize,$blocks)=stat($fname);
+
+if ($size>40000) {
+    print $fh "Rotating logs ...
      ";
+    rename("$fname.2","$fname.3");
+    rename("$fname.1","$fname.2");
+    rename("$fname","$fname.1");
+}
+
+   
+&errout($fh);
 # ---------------------------------------------------------------------- lonnet
 
 print $fh '
      lonnet
      Temp Log
      ';
+print "lonnet\n";
 if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
 open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
 while ($line=) { 
@@ -616,7 +739,7 @@ print $fh "
      ";
 print $fh '
      Connections
      ';
 
 print $fh "";
-foreach $tryserver (keys %hostname) {
+foreach $tryserver (sort(keys(%hostname))) {
 
     $answer=reply("pong",$tryserver);
     if ($answer eq "$tryserver:$perlvar{'lonHostID'}") {
@@ -635,6 +758,7 @@ print $fh "
      ";
 # ------------------------------------------------------------ Delayed messages
 
 print $fh '
      Delayed Messages
      ';
+print "buffers\n";
 
 print $fh '
      Scanning Permanent Log
      ';
 
@@ -671,13 +795,17 @@ print $fh "
      Total Error Count: $total
 $now=time;
 $date=localtime($now);
 print $fh "
      $date ($now)\n";
-
+print "writing done\n";
 }
 
 rename ("$statusdir/newstatus.html","$statusdir/index.html");
 
 if ($totalcount>200) {
-   $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}";
+   print "mailing\n";
+   $emailto="$perlvar{'lonAdmEMail'}";
+   if ($totalcount>600) {
+      $emailto.=",$perlvar{'lonSysEMail'}";
+   }
    $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; 
    system(
  "metasend -b -t $emailto -s '$subj' -f $statusdir/index.html -m text/html");


500 Internal Server Error


      Internal Server Error
      

      The server encountered an internal error or
misconfiguration and was unable to complete
your request.
      

      Please contact the server administrator at 
 root@localhost to inform them of the time this error occurred,
 and the actions you performed just before this error.
      

      More information about this error may be available
in the server error log.