--- loncom/loncron 2010/07/17 20:01:56 1.84 +++ loncom/loncron 2015/09/20 18:31:21 1.103 @@ -2,7 +2,7 @@ # Housekeeping program, started by cron, loncontrol and loncron.pl # -# $Id: loncron,v 1.84 2010/07/17 20:01:56 raeburn Exp $ +# $Id: loncron,v 1.103 2015/09/20 18:31:21 raeburn Exp $ # # Copyright Michigan State University Board of Trustees # @@ -32,6 +32,8 @@ use strict; use lib '/home/httpd/lib/perl/'; use LONCAPA::Configuration; +use LONCAPA::Checksumming; +use LONCAPA; use Apache::lonnet; use Apache::loncommon; @@ -155,12 +157,12 @@ sub checkon_daemon { $errors++; my $kadaemon=$daemon; if ($kadaemon eq 'lonmemcached') { $kadaemon='memcached'; } - &log($fh,'
Killall '.$daemon.': '. + &log($fh,'
Killall '.$daemon.': '. `killall $kadaemon 2>&1`.' - '); sleep 1; &log($fh,unlink($pidfile).' - '. `killall -9 $kadaemon 2>&1`. - '
'); + '
'); &log($fh,"

$daemon not running, trying to start

"); if (&start_daemon($fh,$daemon,$pidfile,$args)) { @@ -269,12 +271,7 @@ sub log_machine_info { &log($fh,"

distprobe

"); &log($fh,"
");
-    open(DSH,"$perlvar{'lonDaemons'}/distprobe |");
-    while (my $line=) { 
-	&log($fh,&encode_entities($line,'<>&"')); 
-	$psproc++;
-    }
-    close(DSH);
+    &log($fh,&encode_entities(&LONCAPA::distro(),'<>&"'));
     &log($fh,"
"); &errout($fh); @@ -288,9 +285,11 @@ sub start_logging { &log($fh,(< + + LON Status Report $perlvar{'lonHostID'} + @@ -301,6 +300,7 @@ sub start_logging {
  • Machine Information
  • Temporary Files
  • Session Tokens
    • +
  • WebDAV Session Tokens
  • httpd
  • lonsql
  • lond
    • @@ -331,15 +331,20 @@ ENDHEADERS "".$role. "".&Apache::lonnet::hostname($id)."\n"); } - &log($fh,"

    Spare Hosts

    \n"); return $fh; } @@ -485,7 +490,7 @@ sub clean_lonIDs { my $since=$now-$mtime; if ($since>$perlvar{'lonExpire'}) { $cleaned++; - &log($fh,"Unlinking $fname
    "); + &log($fh,"Unlinking $fname
    "); unlink("$fname"); } else { $active++; @@ -495,6 +500,32 @@ sub clean_lonIDs { &log($fh,"

    $active open session(s)

    "); } +# ------------------------------------------------ clean out webDAV Session IDs +sub clean_webDAV_sessionIDs { + my ($fh)=@_; + if ($perlvar{'lonRole'} eq 'library') { + &log($fh,'

    WebDAV Session Tokens

    '); + my $cleaned=0; + my $active=0; + my $now = time; + if (-d $perlvar{'lonDAVsessDir'}) { + while (my $fname=<$perlvar{'lonDAVsessDir'}/*>) { + my @stats = stat($fname); + my $since=$now-$stats[9]; + if ($since>$perlvar{'lonExpire'}) { + $cleaned++; + &log($fh,"Unlinking $fname
    "); + unlink("$fname"); + } else { + $active++; + } + } + &log($fh,"

    Cleaned up ".$cleaned." stale webDAV session token(s).

    "); + &log($fh,"

    $active open webDAV session(s)

    "); + } + } +} + # ----------------------------------------------------------- clean out sockets sub clean_sockets { my ($fh)=@_; @@ -514,21 +545,13 @@ sub clean_sockets { # ----------------------------------------------------------------------- httpd sub check_httpd_logs { my ($fh)=@_; - &log($fh,'

    httpd

    Access Log

    ');
-    
-    open (DFH,"tail -n25 /etc/httpd/logs/access_log|");
-    while (my $line=) { &log($fh,&encode_entities($line,'<>&"')) };
-    close (DFH);
-	
-    &log($fh,"

    Error Log

    ");
-	
-    open (DFH,"tail -n25 /etc/httpd/logs/error_log|");
-    while (my $line=) { 
-	&log($fh,"$line");
-	if ($line=~/\[error\]/) { $notices++; } 
+    if (open(PIPE,"./lchttpdlogs|")) {
+        while (my $line=) {
+            &log($fh,$line);
+            if ($line=~/\[error\]/) { $notices++; }
+        }
+        close(PIPE);
     }
-    close (DFH);
-    &log($fh,"
    "); &errout($fh); } @@ -537,7 +560,7 @@ sub check_httpd_logs { sub rotate_lonnet_logs { my ($fh)=@_; &log($fh,'

    lonnet

    Temp Log

    ');
-    print "checking logs\n";
+    print "Checking logs.\n";
     if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
 	open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
 	while (my $line=) { 
@@ -581,7 +604,7 @@ sub rotate_other_logs {
 sub test_connections {
     my ($fh)=@_;
     &log($fh,'
    Connections
    ');
-    print "testing connections\n";
+    print "Testing connections.\n";
     &log($fh,"");
     my ($good,$bad)=(0,0);
     my %hostname = &Apache::lonnet::all_hostnames();
@@ -615,7 +638,7 @@ sub test_connections {
 sub check_delayed_msg {
     my ($fh)=@_;
     &log($fh,'
    Delayed Messages
    ');
-    print "checking buffers\n";
+    print "Checking buffers.\n";
     
     &log($fh,'
    Scanning Permanent Log
    ');
 
@@ -626,7 +649,7 @@ sub check_delayed_msg {
 	my ($time,$sdf,$dserv,$dcmd)=split(/:/,$line);
 	if ($sdf eq 'F') { 
 	    my $local=localtime($time);
-	    &log($fh,"Failed: $time, $dserv, $dcmd
    ");
+	    &log($fh,"Failed: $time, $dserv, $dcmd
    ");
 	    $warnings++;
 	}
 	if ($sdf eq 'S') { $unsend--; }
@@ -634,7 +657,9 @@ sub check_delayed_msg {
     }
 
     &log($fh,"
    Total unsend messages: $unsend
    \n");
-    $warnings=$warnings+5*$unsend;
+    if ($unsend > 0) {
+        $warnings=$warnings+5*$unsend;
+    }
 
     if ($unsend) { $simplestatus{'unsend'}=$unsend; }
     &log($fh,"
    Outgoing Buffer
    \n
    ");
@@ -648,11 +673,28 @@ sub check_delayed_msg {
     }
     &log($fh,"
    \n");
     close (DFH);
+    my %hostname = &Apache::lonnet::all_hostnames();
+    my $numhosts = scalar(keys(%hostname));
 # pong to all servers that have delayed messages
 # this will trigger a reverse connection, which should flush the buffers
-    foreach my $tryserver (keys %servers) {
-	my $answer=&Apache::lonnet::reply("pong",$tryserver);
-	&log($fh,"Pong to $tryserver: $answer
    ");
+    foreach my $tryserver (sort(keys(%servers))) {
+        if ($hostname{$tryserver} || !$numhosts) {
+            my $answer;
+            eval {
+                local $SIG{ ALRM } = sub { die "TIMEOUT" };
+                alarm(20);
+                $answer = &Apache::lonnet::reply("pong",$tryserver);
+                alarm(0);
+            };
+            if ($@ && $@ =~ m/TIMEOUT/) {
+                &log($fh,"Attempted pong to $tryserver timed out
    ");
+                print "Time out while contacting: $tryserver for pong.\n";
+            } else {
+                &log($fh,"Pong to $tryserver: $answer
    ");
+            }
+        } else {
+            &log($fh,"$tryserver has delayed messages, but is not part of the cluster -- skipping 'Pong'.
    ");
+        }
     }
 }
 
@@ -665,7 +707,7 @@ sub finish_logging {
     my $now=time;
     my $date=localtime($now);
     &log($fh,"
    $date ($now)\n");
-    print "lon-status webpage updated\n";
+    print "lon-status webpage updated.\n";
     $fh->close();
 
     if ($errors) { $simplestatus{'errors'}=$errors; }
@@ -686,23 +728,94 @@ sub log_simplestatus {
 }
 
 sub write_loncaparevs {
-    if (open(my $fh,">$perlvar{'lonTabDir'}/loncaparevs.tab")) {
-        my %hostname = &Apache::lonnet::all_hostnames();
-        foreach my $id (sort(keys(%hostname))) {
-            if ($id ne '') {
-                my $loncaparev = &Apache::lonnet::get_server_loncaparev('',$id,1,'loncron');
-                if ($loncaparev =~ /^[\d.\-]+$/) {
-                    print $fh $id.':'.$loncaparev."\n";
+    print "Retrieving LON-CAPA version information.\n";
+    my %hostname = &Apache::lonnet::all_hostnames();
+    my $output;
+    foreach my $id (sort(keys(%hostname))) {
+        if ($id ne '') {
+            my $loncaparev;
+            eval {
+                local $SIG{ ALRM } = sub { die "TIMEOUT" };
+                alarm(10);
+                $loncaparev =
+                    &Apache::lonnet::get_server_loncaparev('',$id,1,'loncron');
+                alarm(0);
+            };
+            if ($@ && $@ =~ m/TIMEOUT/) {
+                print "Time out while contacting lonHost: $id for version.\n";   
+            }
+            if ($loncaparev =~ /^[\w.\-]+$/) {
+                $output .= $id.':'.$loncaparev."\n";
+            }
+        }
+    }
+    if ($output) {
+        if (open(my $fh,">$perlvar{'lonTabDir'}/loncaparevs.tab")) {
+            print $fh $output;
+            close($fh);
+            &Apache::lonnet::load_loncaparevs();
+        }
+    }
+    return;
+}
+
+sub write_serverhomeIDs {
+    print "Retrieving LON-CAPA lonHostID information.\n";
+    my %name_to_host = &Apache::lonnet::all_names();
+    my $output;
+    foreach my $name (sort(keys(%name_to_host))) {
+        if ($name ne '') {
+            if (ref($name_to_host{$name}) eq 'ARRAY') {
+                my $serverhomeID;
+                eval {
+                    local $SIG{ ALRM } = sub { die "TIMEOUT" };
+                    alarm(10);
+                    $serverhomeID = 
+                        &Apache::lonnet::get_server_homeID($name,1,'loncron');
+                    alarm(0);
+                };
+                if ($@ && $@ =~ m/TIMEOUT/) {
+                    print "Time out while contacting server: $name\n"; 
+                }
+                if ($serverhomeID ne '') {
+                    $output .= $name.':'.$serverhomeID."\n";
+                } else {
+                    $output .= $name.':'.$name_to_host{$name}->[0]."\n";
                 }
             }
         }
-        close($fh);
+    }
+    if ($output) {
+        if (open(my $fh,">$perlvar{'lonTabDir'}/serverhomeIDs.tab")) {
+            print $fh $output;
+            close($fh);
+            &Apache::lonnet::load_serverhomeIDs();
+        }
+    }
+    return;
+}
+
+sub write_checksums {
+    my $distro = &LONCAPA::distro();
+    if ($distro) {
+        print "Retrieving file version and checksumming.\n";
+        my $numchksums = 0;
+        my ($chksumsref,$versionsref) =
+            &LONCAPA::Checksumming::get_checksums($distro,$perlvar{'lonDaemons'},
+                                                  $perlvar{'lonLib'},
+                                                  $perlvar{'lonIncludes'},
+                                                  $perlvar{'lonTabDir'});
+        if (ref($chksumsref) eq 'HASH') {
+            $numchksums = scalar(keys(%{$chksumsref}));
+        }
+        print "File version retrieved and checksumming completed for $numchksums files.\n";
+    } else {
+        print "File version retrieval and checksumming skipped - could not determine Linux distro.\n"; 
     }
     return;
 }
 
 sub send_mail {
-    print "sending mail\n";
     my $defdom = $perlvar{'lonDefDomain'};
     my $origmail = $perlvar{'lonAdmEMail'};
     my $emailto = &Apache::loncommon::build_recipient_list(undef,
@@ -710,17 +823,40 @@ sub send_mail {
     if ($totalcount>2500) {
 	$emailto.=",$perlvar{'lonSysEMail'}";
     }
-    my $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; 
-
-    my $result=system("metasend -b -S 4000000 -t $emailto -s '$subj' -f $statusdir/index.html -m text/html >& /dev/null");
-    if ($result != 0) {
-	$result=system("mail -s '$subj' $emailto < $statusdir/index.html");
+    my $from;
+    my $hostname=`/bin/hostname`;
+    chop($hostname);
+    $hostname=~s/[^\w\.]//g;
+    if ($hostname) {
+        $from = 'www@'.$hostname;
+    }
+    my $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices";
+    my $loncronmail = "To: $emailto\n".
+                      "From: $from\n".
+                      "Subject: ".$subj."\n".
+                      "Content-type: text/html\; charset=UTF-8\n".
+                      "MIME-Version: 1.0\n\n";
+    if (open(my $fh,"<$statusdir/index.html")) {
+        while (<$fh>) {
+            $loncronmail .= $_;
+        }
+        close($fh);
+    } else {
+        $loncronmail .= "Failed to read from http://$hostname/lon-status/index.html\n";
+    }
+    $loncronmail .= "\n\n";
+    if (open(my $mailh, "|/usr/lib/sendmail -oi -t -odb")) {
+        print $mailh $loncronmail;
+        close($mailh);
+        print "Sending mail.\n";
+    } else {
+        print "Sending mail failed.\n";
     }
 }
 
 sub usage {
     print(< /dev/null");
 	exit 1;
     }
 
 # -------------------------------------------- Force reload of host information
-    &Apache::lonnet::load_hosts_tab(1);
-    &Apache::lonnet::load_domain_tab(1);
-    &Apache::lonnet::get_iphost(1);
+    my $nomemcache;
+    if ($justcheckdaemons) {
+        $nomemcache=1;
+        my $memcachepidfile="$perlvar{'lonDaemons'}/logs/memcached.pid";
+        my $memcachepid;
+        if (-e $memcachepidfile) {
+            my $memfh=IO::File->new($memcachepidfile);
+            $memcachepid=<$memfh>;
+            chomp($memcachepid);
+            if ($memcachepid =~ /^\d+$/ && kill 0 => $memcachepid) {
+                undef($nomemcache);
+            }
+        }
+    }
+    &Apache::lonnet::load_hosts_tab(1,$nomemcache);
+    &Apache::lonnet::load_domain_tab(1,$nomemcache);
+    &Apache::lonnet::get_iphost(1,$nomemcache);
 
 # ----------------------------------------- Force firewall update for lond port  
 
@@ -801,9 +951,12 @@ sub main () {
                 print $fh "$key\n";
             }
             close($fh);
-            my $execpath = $perlvar{'lonDaemons'}.'/lciptables';
-            system("$execpath $tmpfile");
-            unlink($fh);
+            if (&LONCAPA::try_to_lock('/tmp/lock_lciptables')) {
+                my $execpath = $perlvar{'lonDaemons'}.'/lciptables';
+                system("$execpath $tmpfile");
+                unlink('/tmp/lock_lciptables');  # Remove the lock file. 
+            }
+            unlink($tmpfile);
         }
     }
 
@@ -821,6 +974,7 @@ sub main () {
 	&log_machine_info($fh);
 	&clean_tmp($fh);
 	&clean_lonIDs($fh);
+        &clean_webDAV_sessionIDs($fh);
 	&check_httpd_logs($fh);
 	&rotate_lonnet_logs($fh);
 	&rotate_other_logs($fh);
@@ -846,21 +1000,13 @@ sub main () {
 	&check_delayed_msg($fh);
 	&finish_logging($fh);
 	&log_simplestatus();
-	
-	if ($totalcount>200 && !$noemail) { &send_mail(); }
-    }
-    if (!$justcheckconnections && !$justreload) {
         &write_loncaparevs();
+        &write_serverhomeIDs();
+	&write_checksums();
+	if ($totalcount>200 && !$noemail) { &send_mail(); }
     }
 }
 
 &main();
 1;
 
-
-
-
-
-
-
-


500 Internal Server Error


    Internal Server Error
    

    The server encountered an internal error or
misconfiguration and was unable to complete
your request.
    

    Please contact the server administrator at 
 root@localhost to inform them of the time this error occurred,
 and the actions you performed just before this error.
    

    More information about this error may be available
in the server error log.