--- loncom/loncron 2007/05/31 05:20:18 1.74 +++ loncom/loncron 2014/03/17 14:47:38 1.100 @@ -2,7 +2,7 @@ # Housekeeping program, started by cron, loncontrol and loncron.pl # -# $Id: loncron,v 1.74 2007/05/31 05:20:18 albertel Exp $ +# $Id: loncron,v 1.100 2014/03/17 14:47:38 bisitz Exp $ # # Copyright Michigan State University Board of Trustees # @@ -32,7 +32,10 @@ use strict; use lib '/home/httpd/lib/perl/'; use LONCAPA::Configuration; +use LONCAPA::Checksumming; +use LONCAPA; use Apache::lonnet; +use Apache::loncommon; use IO::File; use IO::Socket; @@ -268,12 +271,7 @@ sub log_machine_info { &log($fh,"

distprobe

"); &log($fh,"
");
-    open(DSH,"$perlvar{'lonDaemons'}/distprobe |");
-    while (my $line=) { 
-	&log($fh,&encode_entities($line,'<>&"')); 
-	$psproc++;
-    }
-    close(DSH);
+    &log($fh,&encode_entities(&LONCAPA::distro(),'<>&"'));
     &log($fh,"
"); &errout($fh); @@ -304,7 +302,6 @@ sub start_logging {
  • lonsql
  • lond
  • lonc
  • -
  • lonhttpd
  • lonnet
  • Connections
  • Delayed Messages
  • @@ -347,33 +344,127 @@ ENDHEADERS sub clean_tmp { my ($fh)=@_; &log($fh,'

    Temporary Files

    '); - my $cleaned=0; - my $old=0; - while (my $fname=<$perlvar{'lonDaemons'}/tmp/*>) { - my ($dev,$ino,$mode,$nlink, - $uid,$gid,$rdev,$size, - $atime,$mtime,$ctime, - $blksize,$blocks)=stat($fname); - my $now=time; - my $since=$now-$mtime; - if ($since>$perlvar{'lonExpire'}) { - my $line=''; - if (open(PROBE,$fname)) { - $line=; - close(PROBE); - } - unless ($line=~/^CHECKOUTTOKEN\&/) { - $cleaned++; - unlink("$fname"); - } else { - if ($since>365*$perlvar{'lonExpire'}) { - $cleaned++; - unlink("$fname"); - } else { $old++; } - } - } + my ($cleaned,$old,$removed) = (0,0,0); + my %errors = ( + dir => [], + file => [], + failopen => [], + ); + my %error_titles = ( + dir => 'failed to remove empty directory:', + file => 'failed to unlike stale file', + failopen => 'failed to open file or directory' + ); + ($cleaned,$old,$removed) = &recursive_clean_tmp('',$cleaned,$old,$removed,\%errors); + &log($fh,"Cleaned up: ".$cleaned." files; removed: $removed empty directories; (found: $old old checkout tokens)"); + foreach my $key (sort(keys(%errors))) { + if (ref($errors{$key}) eq 'ARRAY') { + if (@{$errors{$key}} > 0) { + &log($fh,"Error during cleanup ($error_titles{$key}):
    '); + } + } + } +} + +sub recursive_clean_tmp { + my ($subdir,$cleaned,$old,$removed,$errors) = @_; + my $base = "$perlvar{'lonDaemons'}/tmp"; + my $path = $base; + next if ($subdir =~ m{\.\./}); + next unless (ref($errors) eq 'HASH'); + unless ($subdir eq '') { + $path .= '/'.$subdir; + } + if (opendir(my $dh,"$path")) { + while (my $file = readdir($dh)) { + next if ($file =~ /^\.\.?$/); + my $fname = "$path/$file"; + if (-d $fname) { + my $innerdir; + if ($subdir eq '') { + $innerdir = $file; + } else { + $innerdir = $subdir.'/'.$file; + } + ($cleaned,$old,$removed) = + &recursive_clean_tmp($innerdir,$cleaned,$old,$removed,$errors); + my @doms = &Apache::lonnet::current_machine_domains(); + + if (open(my $dirhandle,$fname)) { + unless (($innerdir eq 'helprequests') || + (($innerdir =~ /^addcourse/) && ($innerdir !~ m{/\d+$}))) { + my @contents = grep {!/^\.\.?$/} readdir($dirhandle); + join('&&',@contents)."\n"; + if (scalar(grep {!/^\.\.?$/} readdir($dirhandle)) == 0) { + closedir($dirhandle); + if ($fname =~ m{^\Q$perlvar{'lonDaemons'}\E/tmp/}) { + if (rmdir($fname)) { + $removed ++; + } elsif (ref($errors->{dir}) eq 'ARRAY') { + push(@{$errors->{dir}},$fname); + } + } + } + } else { + closedir($dirhandle); + } + } + } else { + my ($dev,$ino,$mode,$nlink, + $uid,$gid,$rdev,$size, + $atime,$mtime,$ctime, + $blksize,$blocks)=stat($fname); + my $now=time; + my $since=$now-$mtime; + if ($since>$perlvar{'lonExpire'}) { + if ($subdir eq '') { + my $line=''; + if ($fname =~ /\.db$/) { + if (unlink($fname)) { + $cleaned++; + } elsif (ref($errors->{file}) eq 'ARRAY') { + push(@{$errors->{file}},$fname); + } + } elsif (open(PROBE,$fname)) { + my $line=''; + $line=; + close(PROBE); + if ($line=~/^CHECKOUTTOKEN\&/) { + if ($since>365*$perlvar{'lonExpire'}) { + if (unlink($fname)) { + $cleaned++; + } elsif (ref($errors->{file}) eq 'ARRAY') { + push(@{$errors->{file}},$fname); + } + } else { + $old++; + } + } else { + if (unlink($fname)) { + $cleaned++; + } elsif (ref($errors->{file}) eq 'ARRAY') { + push(@{$errors->{file}},$fname); + } + } + } elsif (ref($errors->{failopen}) eq 'ARRAY') { + push(@{$errors->{failopen}},$fname); + } + } else { + if (unlink($fname)) { + $cleaned++; + } elsif (ref($errors->{file}) eq 'ARRAY') { + push(@{$errors->{file}},$fname); + } + } + } + } + } + closedir($dh); + } elsif (ref($errors->{failopen}) eq 'ARRAY') { + push(@{$errors->{failopen}},$path); } - &log($fh,"Cleaned up ".$cleaned." files (".$old." old checkout tokens)."); + return ($cleaned,$old,$removed); } # ------------------------------------------------------------ clean out lonIDs @@ -408,7 +499,7 @@ sub clean_sockets { opendir(SOCKETS,$perlvar{'lonSockDir'}); while (my $fname=readdir(SOCKETS)) { next if (-d $fname - || $fname=~/(mysqlsock|maximasock|\Q$perlvar{'lonSockDir'}\E)/); + || $fname=~/(mysqlsock|maximasock|rsock|\Q$perlvar{'lonSockDir'}\E)/); $cleaned++; &log($fh,"Unlinking $fname
    "); unlink("/home/httpd/sockets/$fname"); @@ -420,21 +511,13 @@ sub clean_sockets { # ----------------------------------------------------------------------- httpd sub check_httpd_logs { my ($fh)=@_; - &log($fh,'

    httpd

    Access Log

    ');
    -    
    -    open (DFH,"tail -n25 /etc/httpd/logs/access_log|");
    -    while (my $line=) { &log($fh,&encode_entities($line,'<>&"')) };
    -    close (DFH);
    -	
    -    &log($fh,"

    Error Log

    ");
    -	
    -    open (DFH,"tail -n25 /etc/httpd/logs/error_log|");
    -    while (my $line=) { 
    -	&log($fh,"$line");
    -	if ($line=~/\[error\]/) { $notices++; } 
    +    if (open(PIPE,"./lchttpdlogs|")) {
    +        while (my $line=) {
    +            &log($fh,$line);
    +            if ($line=~/\[error\]/) { $notices++; }
    +        }
    +        close(PIPE);
         }
    -    close (DFH);
    -    &log($fh,"
    "); &errout($fh); } @@ -443,7 +526,7 @@ sub check_httpd_logs { sub rotate_lonnet_logs { my ($fh)=@_; &log($fh,'

    lonnet

    Temp Log

    ');
    -    print "checking logs\n";
    +    print "Checking logs.\n";
         if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
     	open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
     	while (my $line=) { 
    @@ -470,19 +553,24 @@ sub rotate_lonnet_logs {
     
     sub rotate_other_logs {
         my ($fh) = @_;
    -    my $fname="$perlvar{'lonDaemons'}/logs/autoenroll.log";
    -    &rotate_logfile($fname,$fh,'Auto Enroll log');
    -    $fname="$perlvar{'lonDaemons'}/logs/autocreate.log";
    -    &rotate_logfile($fname,$fh,'Create Course log');
    -    $fname="$perlvar{'lonDaemons'}/logs/searchcat.log";
    -    &rotate_logfile($fname,$fh,'Search Cataloguing log');
    +    my %logs = (
    +                  autoenroll          => 'Auto Enroll log',
    +                  autocreate          => 'Create Course log',
    +                  searchcat           => 'Search Cataloguing log',
    +                  autoupdate          => 'Auto Update log',
    +                  refreshcourseids_db => 'Refresh CourseIDs db log',
    +               );
    +    foreach my $item (keys(%logs)) {
    +        my $fname=$perlvar{'lonDaemons'}.'/logs/'.$item.'.log';
    +        &rotate_logfile($fname,$fh,$logs{$item});
    +    }
     }
     
     # ----------------------------------------------------------------- Connections
     sub test_connections {
         my ($fh)=@_;
         &log($fh,'

    Connections

    '); - print "testing connections\n"; + print "Testing connections.\n"; &log($fh,""); my ($good,$bad)=(0,0); my %hostname = &Apache::lonnet::all_hostnames(); @@ -516,7 +604,7 @@ sub test_connections { sub check_delayed_msg { my ($fh)=@_; &log($fh,'

    Delayed Messages

    '); - print "checking buffers\n"; + print "Checking buffers.\n"; &log($fh,'

    Scanning Permanent Log

    '); @@ -535,7 +623,9 @@ sub check_delayed_msg { } &log($fh,"

    Total unsend messages: $unsend

    \n"); - $warnings=$warnings+5*$unsend; + if ($unsend > 0) { + $warnings=$warnings+5*$unsend; + } if ($unsend) { $simplestatus{'unsend'}=$unsend; } &log($fh,"

    Outgoing Buffer

    \n
    ");
    @@ -549,11 +639,28 @@ sub check_delayed_msg {
         }
         &log($fh,"
    \n"); close (DFH); + my %hostname = &Apache::lonnet::all_hostnames(); + my $numhosts = scalar(keys(%hostname)); # pong to all servers that have delayed messages # this will trigger a reverse connection, which should flush the buffers - foreach my $tryserver (keys %servers) { - my $answer=&Apache::lonnet::reply("pong",$tryserver); - &log($fh,"Pong to $tryserver: $answer
    "); + foreach my $tryserver (sort(keys(%servers))) { + if ($hostname{$tryserver} || !$numhosts) { + my $answer; + eval { + local $SIG{ ALRM } = sub { die "TIMEOUT" }; + alarm(20); + $answer = &Apache::lonnet::reply("pong",$tryserver); + alarm(0); + }; + if ($@ && $@ =~ m/TIMEOUT/) { + &log($fh,"Attempted pong to $tryserver timed out
    "); + print "Time out while contacting: $tryserver for pong.\n"; + } else { + &log($fh,"Pong to $tryserver: $answer
    "); + } + } else { + &log($fh,"$tryserver has delayed messages, but is not part of the cluster -- skipping 'Pong'.
    "); + } } } @@ -566,7 +673,7 @@ sub finish_logging { my $now=time; my $date=localtime($now); &log($fh,"
    $date ($now)\n"); - print "lon-status webpage updated\n"; + print "lon-status webpage updated.\n"; $fh->close(); if ($errors) { $simplestatus{'errors'}=$errors; } @@ -586,9 +693,100 @@ sub log_simplestatus { $sfh->close(); } +sub write_loncaparevs { + print "Retrieving LON-CAPA version information.\n"; + my %hostname = &Apache::lonnet::all_hostnames(); + my $output; + foreach my $id (sort(keys(%hostname))) { + if ($id ne '') { + my $loncaparev; + eval { + local $SIG{ ALRM } = sub { die "TIMEOUT" }; + alarm(10); + $loncaparev = + &Apache::lonnet::get_server_loncaparev('',$id,1,'loncron'); + alarm(0); + }; + if ($@ && $@ =~ m/TIMEOUT/) { + print "Time out while contacting lonHost: $id for version.\n"; + } + if ($loncaparev =~ /^[\w.\-]+$/) { + $output .= $id.':'.$loncaparev."\n"; + } + } + } + if ($output) { + if (open(my $fh,">$perlvar{'lonTabDir'}/loncaparevs.tab")) { + print $fh $output; + close($fh); + &Apache::lonnet::load_loncaparevs(); + } + } + return; +} + +sub write_serverhomeIDs { + print "Retrieving LON-CAPA lonHostID information.\n"; + my %name_to_host = &Apache::lonnet::all_names(); + my $output; + foreach my $name (sort(keys(%name_to_host))) { + if ($name ne '') { + if (ref($name_to_host{$name}) eq 'ARRAY') { + my $serverhomeID; + eval { + local $SIG{ ALRM } = sub { die "TIMEOUT" }; + alarm(10); + $serverhomeID = + &Apache::lonnet::get_server_homeID($name,1,'loncron'); + alarm(0); + }; + if ($@ && $@ =~ m/TIMEOUT/) { + print "Time out while contacting server: $name\n"; + } + if ($serverhomeID ne '') { + $output .= $name.':'.$serverhomeID."\n"; + } else { + $output .= $name.':'.$name_to_host{$name}->[0]."\n"; + } + } + } + } + if ($output) { + if (open(my $fh,">$perlvar{'lonTabDir'}/serverhomeIDs.tab")) { + print $fh $output; + close($fh); + &Apache::lonnet::load_serverhomeIDs(); + } + } + return; +} + +sub write_checksums { + my $distro = &LONCAPA::distro(); + if ($distro) { + print "Retrieving file version and checksumming.\n"; + my $numchksums = 0; + my ($chksumsref,$versionsref) = + &LONCAPA::Checksumming::get_checksums($distro,$perlvar{'lonDaemons'}, + $perlvar{'lonLib'}, + $perlvar{'lonIncludes'}, + $perlvar{'lonTabDir'}); + if (ref($chksumsref) eq 'HASH') { + $numchksums = scalar(keys(%{$chksumsref})); + } + print "File version retrieved and checksumming completed for $numchksums files.\n"; + } else { + print "File version retrieval and checksumming skipped - could not determine Linux distro.\n"; + } + return; +} + sub send_mail { - print "sending mail\n"; - my $emailto="$perlvar{'lonAdmEMail'}"; + print "Sending mail.\n"; + my $defdom = $perlvar{'lonDefDomain'}; + my $origmail = $perlvar{'lonAdmEMail'}; + my $emailto = &Apache::loncommon::build_recipient_list(undef, + 'lonstatusmail',$defdom,$origmail); if ($totalcount>2500) { $emailto.=",$perlvar{'lonSysEMail'}"; } @@ -602,7 +800,7 @@ sub send_mail { sub usage { print(< /dev/null"); exit 1; } @@ -671,6 +869,27 @@ sub main () { &Apache::lonnet::load_domain_tab(1); &Apache::lonnet::get_iphost(1); +# ----------------------------------------- Force firewall update for lond port + + if ((!$justcheckdaemons) && (!$justreload)) { + my $now = time; + my $tmpfile = $perlvar{'lonDaemons'}.'/tmp/lciptables_iphost_'. + $now.$$.int(rand(10000)); + if (open(my $fh,">$tmpfile")) { + my %iphosts = &Apache::lonnet::get_iphost(); + foreach my $key (keys(%iphosts)) { + print $fh "$key\n"; + } + close($fh); + if (&LONCAPA::try_to_lock('/tmp/lock_lciptables')) { + my $execpath = $perlvar{'lonDaemons'}.'/lciptables'; + system("$execpath $tmpfile"); + unlink('/tmp/lock_lciptables'); # Remove the lock file. + } + unlink($tmpfile); + } + } + # ---------------------------------------------------------------- Start report $errors=0; @@ -690,14 +909,14 @@ sub main () { &rotate_other_logs($fh); } if (!$justcheckconnections && !$justreload) { + &checkon_daemon($fh,'lonmemcached',40000); &checkon_daemon($fh,'lonsql',200000); if ( &checkon_daemon($fh,'lond',40000,'USR1') eq 'running') { &checkon_daemon($fh,'lond',40000,'USR2'); } &checkon_daemon($fh,'lonc',40000,'USR1'); - &checkon_daemon($fh,'lonhttpd',40000); - &checkon_daemon($fh,'lonmemcached',40000); &checkon_daemon($fh,'lonmaxima',40000); + &checkon_daemon($fh,'lonr',40000); } if ($justreload) { &checkon_daemon($fh,'lond',40000,'USR2'); @@ -710,7 +929,9 @@ sub main () { &check_delayed_msg($fh); &finish_logging($fh); &log_simplestatus(); - + &write_loncaparevs(); + &write_serverhomeIDs(); + &write_checksums(); if ($totalcount>200 && !$noemail) { &send_mail(); } } } @@ -718,10 +939,3 @@ sub main () { &main(); 1; - - - - - - -