--- loncom/loncron	2005/02/14 00:29:48	1.57
+++ loncom/loncron	2011/11/14 17:27:34	1.95
@@ -2,7 +2,7 @@
 
 # Housekeeping program, started by cron, loncontrol and loncron.pl
 #
-# $Id: loncron,v 1.57 2005/02/14 00:29:48 albertel Exp $
+# $Id: loncron,v 1.95 2011/11/14 17:27:34 raeburn Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -32,6 +32,9 @@ use strict;
 
 use lib '/home/httpd/lib/perl/';
 use LONCAPA::Configuration;
+use LONCAPA;
+use Apache::lonnet;
+use Apache::loncommon;
 
 use IO::File;
 use IO::Socket;
@@ -43,21 +46,6 @@ use vars qw (%perlvar %simplestatus $err
 my $statusdir="/home/httpd/html/lon-status";
 
 
-# -------------------------------------------------- Non-critical communication
-sub reply {
-    my ($cmd,$server)=@_;
-    my $peerfile="$perlvar{'lonSockDir'}/$server";
-    my $client=IO::Socket::UNIX->new(Peer    =>"$peerfile",
-                                     Type    => SOCK_STREAM,
-                                     Timeout => 10)
-       or return "con_lost";
-    print $client "$cmd\n";
-    my $answer=<$client>;
-    chomp($answer);
-    if (!$answer) { $answer="con_lost"; }
-    return $answer;
-}
-
 # --------------------------------------------------------- Output error status
 
 sub log {
@@ -76,30 +64,36 @@ sub errout {
 ENDERROUT
 }
 
+sub rotate_logfile {
+    my ($file,$fh,$description) = @_;
+    my $size=(stat($file))[7];
+    if ($size>40000) {
+	&log($fh,"<p>Rotating $description ...</p>");
+	rename("$file.2","$file.3");
+	rename("$file.1","$file.2");
+	rename("$file","$file.1");
+    } 
+}
+
 sub start_daemon {
     my ($fh,$daemon,$pidfile,$args) = @_;
     my $progname=$daemon;
-    if ($daemon eq 'lonc' && $args eq 'new') {
+    if ($daemon eq 'lonc') {
 	$progname='loncnew'; 
-	print "new ";
     }
     my $error_fname="$perlvar{'lonDaemons'}/logs/${daemon}_errors";
-    my $size=(stat($error_fname))[7];
-    if ($size>40000) {
-	&log($fh,"<p>Rotating error logs ...</p>");
-	rename("$error_fname.2","$error_fname.3");
-	rename("$error_fname.1","$error_fname.2");
-	rename("$error_fname","$error_fname.1");
+    &rotate_logfile($error_fname,$fh,'error logs');
+    if ($daemon eq 'lonc') {
+	&clean_sockets($fh);
     }
     system("$perlvar{'lonDaemons'}/$progname 2>$perlvar{'lonDaemons'}/logs/${daemon}_errors");
-    sleep 2;
+    sleep 1;
     if (-e $pidfile) {
 	&log($fh,"<p>Seems like it started ...</p>");
 	my $lfh=IO::File->new("$pidfile");
 	my $daemonpid=<$lfh>;
 	chomp($daemonpid);
-	sleep 2;
-	if (kill 0 => $daemonpid) {
+	if ($daemonpid =~ /^\d+$/ && kill 0 => $daemonpid) {
 	    return 1;
 	} else {
 	    return 0;
@@ -111,8 +105,9 @@ sub start_daemon {
 }
 
 sub checkon_daemon {
-    my ($fh,$daemon,$maxsize,$sendusr1,$args)=@_;
+    my ($fh,$daemon,$maxsize,$send,$args)=@_;
 
+    my $result;
     &log($fh,'<hr /><a name="'.$daemon.'" /><h2>'.$daemon.'</h2><h3>Log</h3><p style="white-space: pre;"><tt>');
     printf("%-15s ",$daemon);
     if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){
@@ -135,13 +130,20 @@ sub checkon_daemon {
 	my $lfh=IO::File->new("$pidfile");
 	$daemonpid=<$lfh>;
 	chomp($daemonpid);
-	if (kill 0 => $daemonpid) {
+	if ($daemonpid =~ /^\d+$/ && kill 0 => $daemonpid) {
 	    &log($fh,"<h3>$daemon at pid $daemonpid responding");
-	    if ($sendusr1) { &log($fh,", sending USR1"); }
+	    if ($send) { &log($fh,", sending $send"); }
 	    &log($fh,"</h3>");
-	    if ($sendusr1) { kill USR1 => $daemonpid; }
+	    if ($send eq 'USR1') { kill USR1 => $daemonpid; }
+	    if ($send eq 'USR2') { kill USR2 => $daemonpid; }
 	    $restartflag=0;
-	    print "running\n";
+	    if ($send eq 'USR2') {
+		$result = 'reloaded';
+		print "reloaded\n";
+	    } else {
+		$result = 'running';
+		print "running\n";
+	    }
 	} else {
 	    $errors++;
 	    &log($fh,"<h3>$daemon at pid $daemonpid not responding</h3>");
@@ -156,7 +158,7 @@ sub checkon_daemon {
 	if ($kadaemon eq 'lonmemcached') { $kadaemon='memcached'; }
 	&log($fh,'<br><font color="red">Killall '.$daemon.': '.
 	    `killall $kadaemon 2>&1`.' - ');
-	sleep 2;
+	sleep 1;
 	&log($fh,unlink($pidfile).' - '.
 	    `killall -9 $kadaemon 2>&1`.
 	    '</font><br>');
@@ -165,6 +167,7 @@ sub checkon_daemon {
 	if (&start_daemon($fh,$daemon,$pidfile,$args)) {
 	    &log($fh,"<h3>$daemon at pid $daemonpid responding</h3>");
 	    $simplestatus{$daemon}='restarted';
+	    $result = 'started';
 	    print "started\n";
 	} else {
 	    $errors++;
@@ -174,8 +177,10 @@ sub checkon_daemon {
 	    if (&start_daemon($fh,$daemon,$pidfile,$args)) {
 		&log($fh,"<h3>$daemon at pid $daemonpid responding</h3>");
 		$simplestatus{$daemon}='restarted';
+		$result = 'started';
 		print "started\n";
 	    } else {
+		$result = 'failed';
 		print " failed\n";
 		$simplestatus{$daemon}='failed';
 		$errors++; $errors++;
@@ -198,20 +203,10 @@ sub checkon_daemon {
     }
     
     my $fname="$perlvar{'lonDaemons'}/logs/$daemon.log";
-    
-    my ($dev,$ino,$mode,$nlink,
-	$uid,$gid,$rdev,$size,
-	$atime,$mtime,$ctime,
-	$blksize,$blocks)=stat($fname);
-    
-    if ($size>$maxsize) {
-	&log($fh,"<p>Rotating logs ...</p>");
-	rename("$fname.2","$fname.3");
-	rename("$fname.1","$fname.2");
-	rename("$fname","$fname.1");
-    }
+    &rotate_logfile($fname,$fh,'logs');
 
     &errout($fh);
+    return $result;
 }
 
 # --------------------------------------------------------------------- Machine
@@ -273,11 +268,20 @@ sub log_machine_info {
     if ($psproc>200) { $notices++; }
     if ($psproc>250) { $notices++; }
 
+    &log($fh,"<h3>distprobe</h3>");
+    &log($fh,"<pre>");
+    open(DSH,"$perlvar{'lonDaemons'}/distprobe |");
+    while (my $line=<DSH>) { 
+	&log($fh,&encode_entities($line,'<>&"')); 
+	$psproc++;
+    }
+    close(DSH);
+    &log($fh,"</pre>");
+
     &errout($fh);
 }
 
 sub start_logging {
-    my ($hostdom,$hostrole,$hostname,$spareid)=@_;
     my $fh=IO::File->new(">$statusdir/newstatus.html");
     my %simplestatus=();
     my $now=time;
@@ -302,7 +306,6 @@ sub start_logging {
 <li><a href="#lonsql">lonsql</a></li>
 <li><a href="#lond">lond</a></li>
 <li><a href="#lonc">lonc</a></li>
-<li><a href="#lonhttpd">lonhttpd</a></li>
 <li><a href="#lonnet">lonnet</a></li>
 <li><a href="#connections">Connections</a></li>
 <li><a href="#delayed">Delayed Messages</a></li>
@@ -320,17 +323,24 @@ ENDHEADERS
 	     &encode_entities($perlvar{$varname},'<>&"')."</td></tr>\n");
     }
     &log($fh,"</table><h3>Hosts</h3><table border='2'>");
-    foreach my $id (sort(keys(%{$hostname}))) {
+    my %hostname = &Apache::lonnet::all_hostnames();
+    foreach my $id (sort(keys(%hostname))) {
+	my $role = (&Apache::lonnet::is_library($id) ? 'library'
+		                                     : 'access');
 	&log($fh,
-	    "<tr><td>$id</td><td>".$hostdom->{$id}.
-	    "</td><td>".$hostrole->{$id}.
-	    "</td><td>".$hostname->{$id}."</td></tr>\n");
-    }
-    &log($fh,"</table><h3>Spare Hosts</h3><ol>");
-    foreach my $id (sort(keys(%{$spareid}))) {
-	&log($fh,"<li>$id\n</li>");
+	    "<tr><td>$id</td><td>".&Apache::lonnet::host_domain($id).
+	    "</td><td>".$role.
+	    "</td><td>".&Apache::lonnet::hostname($id)."</td></tr>\n");
+    }
+    &log($fh,"</table><h3>Spare Hosts</h3><ul>");
+    foreach my $type (sort(keys(%Apache::lonnet::spareid))) {
+	&log($fh,"<li>$type\n<ol>");
+	foreach my $id (@{ $Apache::lonnet::spareid{$type} }) {
+	    &log($fh,"<li>$id</li>\n");
+	}
+	&log($fh,"</ol>\n</li>\n");
     }
-    &log($fh,"</ol>\n");
+    &log($fh,"</ul>\n");
     return $fh;
 }
 
@@ -338,33 +348,127 @@ ENDHEADERS
 sub clean_tmp {
     my ($fh)=@_;
     &log($fh,'<hr /><a name="tmp" /><h2>Temporary Files</h2>');
-    my $cleaned=0;
-    my $old=0;
-    while (my $fname=<$perlvar{'lonDaemons'}/tmp/*>) {
-	my ($dev,$ino,$mode,$nlink,
-	    $uid,$gid,$rdev,$size,
-	    $atime,$mtime,$ctime,
-	    $blksize,$blocks)=stat($fname);
-	my $now=time;
-	my $since=$now-$mtime;
-	if ($since>$perlvar{'lonExpire'}) {
-	    my $line='';
-	    if (open(PROBE,$fname)) {
-		$line=<PROBE>;
-		close(PROBE);
-	    }
-	    unless ($line=~/^CHECKOUTTOKEN\&/) {
-		$cleaned++;
-		unlink("$fname");
-	    } else {
-		if ($since>365*$perlvar{'lonExpire'}) {
-		    $cleaned++;
-		    unlink("$fname");
-		} else { $old++; }
-	    }
-	}
+    my ($cleaned,$old,$removed) = (0,0,0);
+    my %errors = (
+                     dir       => [],
+                     file      => [],
+                     failopen  => [],
+                 );
+    my %error_titles = (
+                         dir       => 'failed to remove empty directory:',
+                         file      => 'failed to unlike stale file',
+                         failopen  => 'failed to open file or directory'
+                       );
+    ($cleaned,$old,$removed) = &recursive_clean_tmp('',$cleaned,$old,$removed,\%errors);
+    &log($fh,"Cleaned up: ".$cleaned." files; removed: $removed empty directories; (found: $old old checkout tokens)");
+    foreach my $key (sort(keys(%errors))) {
+        if (ref($errors{$key}) eq 'ARRAY') {
+            if (@{$errors{$key}} > 0) {
+                &log($fh,"Error during cleanup ($error_titles{$key}):<ul><li>".
+                     join('</li><li><tt>',@{$errors{$key}}).'</tt></li></ul><br />');
+            }
+        }
+    }
+}
+
+sub recursive_clean_tmp {
+    my ($subdir,$cleaned,$old,$removed,$errors) = @_;
+    my $base = "$perlvar{'lonDaemons'}/tmp";
+    my $path = $base;
+    next if ($subdir =~ m{\.\./});
+    next unless (ref($errors) eq 'HASH');
+    unless ($subdir eq '') {
+        $path .= '/'.$subdir;
+    }
+    if (opendir(my $dh,"$path")) {
+        while (my $file = readdir($dh)) {
+            next if ($file =~ /^\.\.?$/);
+            my $fname = "$path/$file";
+            if (-d $fname) {
+                my $innerdir;
+                if ($subdir eq '') {
+                    $innerdir = $file;
+                } else {
+                    $innerdir = $subdir.'/'.$file;
+                }
+                ($cleaned,$old,$removed) = 
+                     &recursive_clean_tmp($innerdir,$cleaned,$old,$removed,$errors);
+                my @doms = &Apache::lonnet::current_machine_domains();
+                
+                if (open(my $dirhandle,$fname)) {
+                    unless (($innerdir eq 'helprequests') ||
+                            (($innerdir =~ /^addcourse/) && ($innerdir !~ m{/\d+$}))) {
+                        my @contents = grep {!/^\.\.?$/} readdir($dirhandle);
+                                      join('&&',@contents)."\n";    
+                        if (scalar(grep {!/^\.\.?$/} readdir($dirhandle)) == 0) {
+                            closedir($dirhandle);
+                            if ($fname =~ m{^\Q$perlvar{'lonDaemons'}\E/tmp/}) {
+                                if (rmdir($fname)) {
+                                    $removed ++;
+                                } elsif (ref($errors->{dir}) eq 'ARRAY') {
+                                    push(@{$errors->{dir}},$fname);
+                                }
+                            }
+                        }
+                    } else {
+                        closedir($dirhandle);
+                    }
+                }
+            } else {
+                my ($dev,$ino,$mode,$nlink,
+                    $uid,$gid,$rdev,$size,
+                    $atime,$mtime,$ctime,
+                    $blksize,$blocks)=stat($fname);
+                my $now=time;
+                my $since=$now-$mtime;
+                if ($since>$perlvar{'lonExpire'}) {
+                    if ($subdir eq '') {
+                        my $line='';
+                        if ($fname =~ /\.db$/) {
+                            if (unlink($fname)) {
+                                $cleaned++;
+                            } elsif (ref($errors->{file}) eq 'ARRAY') {
+                                push(@{$errors->{file}},$fname);
+                            }
+                        } elsif (open(PROBE,$fname)) {
+                            my $line='';
+                            $line=<PROBE>;
+                            close(PROBE);
+                            if ($line=~/^CHECKOUTTOKEN\&/) {
+                                if ($since>365*$perlvar{'lonExpire'}) {
+                                    if (unlink($fname)) {
+                                        $cleaned++; 
+                                    } elsif (ref($errors->{file}) eq 'ARRAY') {
+                                        push(@{$errors->{file}},$fname);
+                                    }
+                                } else {
+                                    $old++;
+                                }
+                            } else {
+                                if (unlink($fname)) {
+                                    $cleaned++;
+                                } elsif (ref($errors->{file}) eq 'ARRAY') {
+                                    push(@{$errors->{file}},$fname);
+                                }
+                            }
+                        } elsif (ref($errors->{failopen}) eq 'ARRAY') {
+                            push(@{$errors->{failopen}},$fname); 
+                        }
+                    } else {
+                        if (unlink($fname)) {
+                            $cleaned++;
+                        } elsif (ref($errors->{file}) eq 'ARRAY') {
+                            push(@{$errors->{file}},$fname);
+                        }
+                    }
+                }
+            }
+        }
+        closedir($dh);
+    } elsif (ref($errors->{failopen}) eq 'ARRAY') {
+        push(@{$errors->{failopen}},$path);
     }
-    &log($fh,"Cleaned up ".$cleaned." files (".$old." old checkout tokens).");
+    return ($cleaned,$old,$removed);
 }
 
 # ------------------------------------------------------------ clean out lonIDs
@@ -392,25 +496,32 @@ sub clean_lonIDs {
     &log($fh,"<h3>$active open session(s)</h3>");
 }
 
+# ----------------------------------------------------------- clean out sockets
+sub clean_sockets {
+    my ($fh)=@_;
+    my $cleaned=0;
+    opendir(SOCKETS,$perlvar{'lonSockDir'});
+    while (my $fname=readdir(SOCKETS)) {
+	next if (-d $fname 
+		 || $fname=~/(mysqlsock|maximasock|rsock|\Q$perlvar{'lonSockDir'}\E)/);
+	$cleaned++;
+	&log($fh,"Unlinking $fname<br />");
+	unlink("/home/httpd/sockets/$fname");
+    }
+    &log($fh,"<p>Cleaned up ".$cleaned." stale sockets.</p>");
+}
+
 
 # ----------------------------------------------------------------------- httpd
 sub check_httpd_logs {
     my ($fh)=@_;
-    &log($fh,'<hr /><a name="httpd" /><h2>httpd</h2><h3>Access Log</h3><pre>');
-    
-    open (DFH,"tail -n25 /etc/httpd/logs/access_log|");
-    while (my $line=<DFH>) { &log($fh,&encode_entities($line,'<>&"')) };
-    close (DFH);
-	
-    &log($fh,"</pre><h3>Error Log</h3><pre>");
-	
-    open (DFH,"tail -n25 /etc/httpd/logs/error_log|");
-    while (my $line=<DFH>) { 
-	&log($fh,"$line");
-	if ($line=~/\[error\]/) { $notices++; } 
+    if (open(PIPE,"./lchttpdlogs|")) {
+        while (my $line=<PIPE>) {
+            &log($fh,$line);
+            if ($line=~/\[error\]/) { $notices++; }
+        }
+        close(PIPE);
     }
-    close (DFH);
-    &log($fh,"</pre>");
     &errout($fh);
 }
 
@@ -438,34 +549,39 @@ sub rotate_lonnet_logs {
     } else { &log($fh,"No perm log\n") }
 
     my $fname="$perlvar{'lonDaemons'}/logs/lonnet.log";
-
-    my ($dev,$ino,$mode,$nlink,
-	$uid,$gid,$rdev,$size,
-	$atime,$mtime,$ctime,
-	$blksize,$blocks)=stat($fname);
-
-    if ($size>40000) {
-	&log($fh,"<p>Rotating logs ...</p>");
-	rename("$fname.2","$fname.3");
-	rename("$fname.1","$fname.2");
-	rename("$fname","$fname.1");
-    }
+    &rotate_logfile($fname,$fh,'lonnet log');
 
     &log($fh,"</pre>");
     &errout($fh);
 }
 
+sub rotate_other_logs {
+    my ($fh) = @_;
+    my %logs = (
+                  autoenroll          => 'Auto Enroll log',
+                  autocreate          => 'Create Course log',
+                  searchcat           => 'Search Cataloguing log',
+                  autoupdate          => 'Auto Update log',
+                  refreshcourseids_db => 'Refresh CourseIDs db log',
+               );
+    foreach my $item (keys(%logs)) {
+        my $fname=$perlvar{'lonDaemons'}.'/logs/'.$item.'.log';
+        &rotate_logfile($fname,$fh,$logs{$item});
+    }
+}
+
 # ----------------------------------------------------------------- Connections
 sub test_connections {
-    my ($fh,$hostname)=@_;
+    my ($fh)=@_;
     &log($fh,'<hr /><a name="connections" /><h2>Connections</h2>');
     print "testing connections\n";
     &log($fh,"<table border='2'>");
     my ($good,$bad)=(0,0);
-    foreach my $tryserver (sort(keys(%{$hostname}))) {
+    my %hostname = &Apache::lonnet::all_hostnames();
+    foreach my $tryserver (sort(keys(%hostname))) {
 	print(".");
 	my $result;
-	my $answer=reply("ping",$tryserver);
+	my $answer=&Apache::lonnet::reply("ping",$tryserver);
 	if ($answer eq "$tryserver:$perlvar{'lonHostID'}") {
 	    $result="<b>ok</b>";
 	    $good++;
@@ -511,17 +627,45 @@ sub check_delayed_msg {
     }
 
     &log($fh,"<p>Total unsend messages: <b>$unsend</b></p>\n");
-    $warnings=$warnings+5*$unsend;
+    if ($unsend > 0) {
+        $warnings=$warnings+5*$unsend;
+    }
 
     if ($unsend) { $simplestatus{'unsend'}=$unsend; }
     &log($fh,"<h3>Outgoing Buffer</h3>\n<pre>");
-
+# list directory with delayed messages and remember offline servers
+    my %servers=();
     open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|");
-    while (my $line=<DFH>) { 
+    while (my $line=<DFH>) {
+        my ($server)=($line=~/\.(\w+)$/);
+        if ($server) { $servers{$server}=1; }
 	&log($fh,&encode_entities($line,'<>&"'));
     }
     &log($fh,"</pre>\n");
     close (DFH);
+    my %hostname = &Apache::lonnet::all_hostnames();
+    my $numhosts = scalar(keys(%hostname));
+# pong to all servers that have delayed messages
+# this will trigger a reverse connection, which should flush the buffers
+    foreach my $tryserver (sort(keys(%servers))) {
+        if ($hostname{$tryserver} || !$numhosts) {
+            my $answer;
+            eval {
+                local $SIG{ ALRM } = sub { die "TIMEOUT" };
+                alarm(20);
+                $answer = &Apache::lonnet::reply("pong",$tryserver);
+                alarm(0);
+            };
+            if ($@ && $@ =~ m/TIMEOUT/) {
+                &log($fh,"Attempted pong to $tryserver timed out<br />");
+                print "time out while contacting: $tryserver for pong\n";
+            } else {
+                &log($fh,"Pong to $tryserver: $answer<br />");
+            }
+        } else {
+            &log($fh,"$tryserver has delayed messages, but is not part of the cluster -- skipping 'Pong'.<br />");
+        }
+    }
 }
 
 sub finish_logging {
@@ -543,7 +687,7 @@ sub finish_logging {
 }
 
 sub log_simplestatus {
-    rename ("$statusdir/newstatus.html","$statusdir/index.html");
+    rename("$statusdir/newstatus.html","$statusdir/index.html");
     
     my $sfh=IO::File->new(">$statusdir/loncron_simple.txt");
     foreach (keys %simplestatus) {
@@ -553,15 +697,76 @@ sub log_simplestatus {
     $sfh->close();
 }
 
+sub write_loncaparevs {
+    print "Retrieving LON-CAPA version information\n";
+    if (open(my $fh,">$perlvar{'lonTabDir'}/loncaparevs.tab")) {
+        my %hostname = &Apache::lonnet::all_hostnames();
+        foreach my $id (sort(keys(%hostname))) {
+            if ($id ne '') {
+                my $loncaparev;
+                eval {
+                    local $SIG{ ALRM } = sub { die "TIMEOUT" };
+                    alarm(10);
+                    $loncaparev =
+                        &Apache::lonnet::get_server_loncaparev('',$id,1,'loncron');
+                    alarm(0);
+                };
+                if ($@ && $@ =~ m/TIMEOUT/) {
+                    print "time out while contacting lonHost: $id for version\n";   
+                }
+                if ($loncaparev =~ /^[\w.\-]+$/) {
+                    print $fh $id.':'.$loncaparev."\n";
+                }
+            }
+        }
+        close($fh);
+    }
+    return;
+}
+
+sub write_serverhomeIDs {
+    print "Retrieving LON-CAPA lonHostID information\n";
+    if (open(my $fh,">$perlvar{'lonTabDir'}/serverhomeIDs.tab")) {
+        my %name_to_host = &Apache::lonnet::all_names();
+        foreach my $name (sort(keys(%name_to_host))) {
+            if ($name ne '') {
+                if (ref($name_to_host{$name}) eq 'ARRAY') {
+                    my $serverhomeID;
+                    eval {
+                        local $SIG{ ALRM } = sub { die "TIMEOUT" };
+                        alarm(10);
+                        $serverhomeID = 
+                            &Apache::lonnet::get_server_homeID($name,1,'loncron');
+                        alarm(0);
+                    };
+                    if ($@ && $@ =~ m/TIMEOUT/) {
+                        print "Time out while contacting server: $name\n"; 
+                    }
+                    if ($serverhomeID ne '') {
+                        print $fh $name.':'.$serverhomeID."\n";
+                    } else {
+                        print $fh $name.':'.$name_to_host{$name}->[0]."\n";
+                    }
+                }
+            }
+        }
+        close($fh);
+    }
+    return;
+}
+
 sub send_mail {
     print "sending mail\n";
-    my $emailto="$perlvar{'lonAdmEMail'}";
+    my $defdom = $perlvar{'lonDefDomain'};
+    my $origmail = $perlvar{'lonAdmEMail'};
+    my $emailto = &Apache::loncommon::build_recipient_list(undef,
+                                   'lonstatusmail',$defdom,$origmail);
     if ($totalcount>2500) {
 	$emailto.=",$perlvar{'lonSysEMail'}";
     }
     my $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; 
 
-    my $result=system("metasend -b -t $emailto -s '$subj' -f $statusdir/index.html -m text/html >& /dev/null");
+    my $result=system("metasend -b -S 4000000 -t $emailto -s '$subj' -f $statusdir/index.html -m text/html >& /dev/null");
     if ($result != 0) {
 	$result=system("mail -s '$subj' $emailto < $statusdir/index.html");
     }
@@ -572,8 +777,7 @@ sub usage {
 loncron - housekeeping program that checks up on various parts of Lon-CAPA
 
 Options:
-   --help     Display help
-   --oldlonc  When starting the lonc daemon use 'lonc' not 'loncnew'
+   --help     Display 
    --noemail  Do not send the status email
    --justcheckconnections  Only check the current status of the lonc/d
                                 connections, do not send emails do not
@@ -583,18 +787,23 @@ Options:
                                 running, do not send emails do not
                                 check the lonc/d connections, do not
                                 generate lon-status
+   --justreload            Only tell the daemons to reload the config files,
+				do not send emails do not
+                                check if the daemons are running, do not
+                                generate lon-status
                            
 USAGE
 }
 
 # ================================================================ Main Program
 sub main () {
-    my ($oldlonc,$help,$justcheckdaemons,$noemail,$justcheckconnections);
+    my ($help,$justcheckdaemons,$noemail,$justcheckconnections,
+	$justreload);
     &GetOptions("help"                 => \$help,
-		"oldlonc"              => \$oldlonc,
 		"justcheckdaemons"     => \$justcheckdaemons,
 		"noemail"              => \$noemail,
-		"justcheckconnections" => \$justcheckconnections
+		"justcheckconnections" => \$justcheckconnections,
+		"justreload"           => \$justreload
 		);
     if ($help) { &usage(); return; }
 # --------------------------------- Read loncapa_apache.conf and loncapa.conf
@@ -603,7 +812,7 @@ sub main () {
     undef $perlvarref;
     delete $perlvar{'lonReceipt'}; # remove since sensitive and not needed
     delete $perlvar{'lonSqlAccess'}; # remove since sensitive and not needed
-
+    chdir($perlvar{'lonDaemons'});
 # --------------------------------------- Make sure that LON-CAPA is configured
 # I only test for one thing here (lonHostID).  This is just a safeguard.
     if ('{[[[[lonHostID]]]]}' eq $perlvar{'lonHostID'}) {
@@ -629,32 +838,31 @@ sub main () {
 	exit 1;
     }
 
-# ------------------------------------------------------------- Read hosts file
-    my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab");
-    
-    my (%hostname,%hostdom,%hostrole,%spareid);
-    while (my $configline=<$config>) {
-	next if ($configline =~ /^(\#|\s*\$)/);
-	my ($id,$domain,$role,$name)=split(/:/,$configline);
-	if ($id && $domain && $role && $name) {
-	    $name=~s/\s//g;
-	    $hostname{$id}=$name;
-	    $hostdom{$id}=$domain;
-	    $hostrole{$id}=$role;
-	}
+# -------------------------------------------- Force reload of host information
+    &Apache::lonnet::load_hosts_tab(1);
+    &Apache::lonnet::load_domain_tab(1);
+    &Apache::lonnet::get_iphost(1);
+
+# ----------------------------------------- Force firewall update for lond port  
+
+    if ((!$justcheckdaemons) && (!$justreload)) {
+        my $now = time;
+        my $tmpfile = $perlvar{'lonDaemons'}.'/tmp/lciptables_iphost_'.
+                      $now.$$.int(rand(10000));
+        if (open(my $fh,">$tmpfile")) {
+            my %iphosts = &Apache::lonnet::get_iphost();
+            foreach my $key (keys(%iphosts)) {
+                print $fh "$key\n";
+            }
+            close($fh);
+            if (&LONCAPA::try_to_lock('/tmp/lock_lciptables')) {
+                my $execpath = $perlvar{'lonDaemons'}.'/lciptables';
+                system("$execpath $tmpfile");
+                unlink('/tmp/lock_lciptables');  # Remove the lock file. 
+            }
+            unlink($tmpfile);
+        }
     }
-    undef $config;
-
-# ------------------------------------------------------ Read spare server file
-    $config=IO::File->new("$perlvar{'lonTabDir'}/spare.tab");
-    
-    while (my $configline=<$config>) {
-	chomp($configline);
-	if (($configline) && ($configline ne $perlvar{'lonHostID'})) {
-	    $spareid{$configline}=1;
-	}
-    }
-    undef $config;
 
 # ---------------------------------------------------------------- Start report
 
@@ -664,31 +872,39 @@ sub main () {
 
 	
     my $fh;
-    if (!$justcheckdaemons && !$justcheckconnections) {
-	$fh=&start_logging(\%hostdom,\%hostrole,\%hostname,\%spareid);
+    if (!$justcheckdaemons && !$justcheckconnections && !$justreload) {
+	$fh=&start_logging();
 
 	&log_machine_info($fh);
 	&clean_tmp($fh);
 	&clean_lonIDs($fh);
 	&check_httpd_logs($fh);
 	&rotate_lonnet_logs($fh);
+	&rotate_other_logs($fh);
     }
-    if (!$justcheckconnections) {
-	&checkon_daemon($fh,'lonsql',200000);
-	&checkon_daemon($fh,'lond',40000,1);
-	my $args='new';
-	if ($oldlonc) { $args = ''; }
-	&checkon_daemon($fh,'lonc',40000,1,$args);
-	&checkon_daemon($fh,'lonhttpd',40000);
+    if (!$justcheckconnections && !$justreload) {
 	&checkon_daemon($fh,'lonmemcached',40000);
+	&checkon_daemon($fh,'lonsql',200000);
+	if ( &checkon_daemon($fh,'lond',40000,'USR1') eq 'running') {
+	    &checkon_daemon($fh,'lond',40000,'USR2');
+	}
+	&checkon_daemon($fh,'lonc',40000,'USR1');
+        &checkon_daemon($fh,'lonmaxima',40000);
+        &checkon_daemon($fh,'lonr',40000);
+    }
+    if ($justreload) {
+	&checkon_daemon($fh,'lond',40000,'USR2');
+	&checkon_daemon($fh,'lonc',40000,'USR2');
     }
-    if (!$justcheckdaemons) {
-	&test_connections($fh,\%hostname);
+    if ($justcheckconnections) {
+	&test_connections($fh);
     }
-    if (!$justcheckdaemons && !$justcheckconnections) {
+    if (!$justcheckdaemons && !$justcheckconnections && !$justreload) {
 	&check_delayed_msg($fh);
 	&finish_logging($fh);
 	&log_simplestatus();
+        &write_loncaparevs();
+        &write_serverhomeIDs();
 	
 	if ($totalcount>200 && !$noemail) { &send_mail(); }
     }