--- loncom/loncron 2004/05/25 15:20:31 1.51 +++ loncom/loncron 2006/01/28 10:03:37 1.65 @@ -2,7 +2,7 @@ # Housekeeping program, started by cron, loncontrol and loncron.pl # -# $Id: loncron,v 1.51 2004/05/25 15:20:31 albertel Exp $ +# $Id: loncron,v 1.65 2006/01/28 10:03:37 albertel Exp $ # # Copyright Michigan State University Board of Trustees # @@ -45,13 +45,13 @@ my $statusdir="/home/httpd/html/lon-stat # -------------------------------------------------- Non-critical communication sub reply { - my ($cmd,$server)=@_; - my $peerfile="$perlvar{'lonSockDir'}/$server"; + my ($cmd,$server,$hostname)=@_; + my $peerfile="$perlvar{'lonSockDir'}/".$hostname->{$server}; my $client=IO::Socket::UNIX->new(Peer =>"$peerfile", Type => SOCK_STREAM, Timeout => 10) or return "con_lost"; - print $client "$cmd\n"; + print $client "sethost:$server:$cmd\n"; my $answer=<$client>; chomp($answer); if (!$answer) { $answer="con_lost"; } @@ -92,14 +92,13 @@ sub start_daemon { rename("$error_fname","$error_fname.1"); } system("$perlvar{'lonDaemons'}/$progname 2>$perlvar{'lonDaemons'}/logs/${daemon}_errors"); - sleep 2; + sleep 1; if (-e $pidfile) { &log($fh,"

Seems like it started ...

"); my $lfh=IO::File->new("$pidfile"); my $daemonpid=<$lfh>; chomp($daemonpid); - sleep 2; - if (kill 0 => $daemonpid) { + if ($daemonpid =~ /^\d+$/ && kill 0 => $daemonpid) { return 1; } else { return 0; @@ -111,10 +110,11 @@ sub start_daemon { } sub checkon_daemon { - my ($fh,$daemon,$maxsize,$sendusr1,$args)=@_; + my ($fh,$daemon,$maxsize,$send,$args)=@_; + my $result; &log($fh,'

'.$daemon.'

Log

'); - printf("%-10s ",$daemon); + printf("%-15s ",$daemon); if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ open (DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/$daemon.log|"); while (my $line=) { @@ -135,13 +135,20 @@ sub checkon_daemon { my $lfh=IO::File->new("$pidfile"); $daemonpid=<$lfh>; chomp($daemonpid); - if (kill 0 => $daemonpid) { + if ($daemonpid =~ /^\d+$/ && kill 0 => $daemonpid) { &log($fh,"

$daemon at pid $daemonpid responding"); - if ($sendusr1) { &log($fh,", sending USR1"); } + if ($send) { &log($fh,", sending $send"); } &log($fh,"

"); - if ($sendusr1) { kill USR1 => $daemonpid; } + if ($send eq 'USR1') { kill USR1 => $daemonpid; } + if ($send eq 'USR2') { kill USR2 => $daemonpid; } $restartflag=0; - print "running\n"; + if ($send eq 'USR2') { + $result = 'reloaded'; + print "reloaded\n"; + } else { + $result = 'running'; + print "running\n"; + } } else { $errors++; &log($fh,"

$daemon at pid $daemonpid not responding

"); @@ -152,17 +159,20 @@ sub checkon_daemon { if ($restartflag==1) { $simplestatus{$daemon}='off'; $errors++; + my $kadaemon=$daemon; + if ($kadaemon eq 'lonmemcached') { $kadaemon='memcached'; } &log($fh,'
Killall '.$daemon.': '. - `killall $daemon 2>&1`.' - '); - sleep 2; + `killall $kadaemon 2>&1`.' - '); + sleep 1; &log($fh,unlink($pidfile).' - '. - `killall -9 $daemon 2>&1`. + `killall -9 $kadaemon 2>&1`. '
'); &log($fh,"

$daemon not running, trying to start

"); if (&start_daemon($fh,$daemon,$pidfile,$args)) { &log($fh,"

$daemon at pid $daemonpid responding

"); $simplestatus{$daemon}='restarted'; + $result = 'started'; print "started\n"; } else { $errors++; @@ -172,8 +182,10 @@ sub checkon_daemon { if (&start_daemon($fh,$daemon,$pidfile,$args)) { &log($fh,"

$daemon at pid $daemonpid responding

"); $simplestatus{$daemon}='restarted'; + $result = 'started'; print "started\n"; } else { + $result = 'failed'; print " failed\n"; $simplestatus{$daemon}='failed'; $errors++; $errors++; @@ -210,6 +222,7 @@ sub checkon_daemon { } &errout($fh); + return $result; } # --------------------------------------------------------------------- Machine @@ -260,7 +273,7 @@ sub log_machine_info { &log($fh,"
");
     my $psproc=0;
 
-    open (PSH,"ps -aux --cols 140 |");
+    open (PSH,"ps aux --cols 140 |");
     while (my $line=) { 
 	&log($fh,&encode_entities($line,'<>&"')); 
 	$psproc++;
@@ -271,6 +284,16 @@ sub log_machine_info {
     if ($psproc>200) { $notices++; }
     if ($psproc>250) { $notices++; }
 
+    &log($fh,"

distprobe

"); + &log($fh,"
");
+    open(DSH,"$perlvar{'lonDaemons'}/distprobe |");
+    while (my $line=) { 
+	&log($fh,&encode_entities($line,'<>&"')); 
+	$psproc++;
+    }
+    close(DSH);
+    &log($fh,"
"); + &errout($fh); } @@ -463,7 +486,7 @@ sub test_connections { foreach my $tryserver (sort(keys(%{$hostname}))) { print("."); my $result; - my $answer=reply("pong",$tryserver); + my $answer=reply("ping",$tryserver,$hostname); if ($answer eq "$tryserver:$perlvar{'lonHostID'}") { $result="ok"; $good++; @@ -554,11 +577,15 @@ sub log_simplestatus { sub send_mail { print "sending mail\n"; my $emailto="$perlvar{'lonAdmEMail'}"; - if ($totalcount>1000) { + if ($totalcount>2500) { $emailto.=",$perlvar{'lonSysEMail'}"; } my $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; - system("metasend -b -t $emailto -s '$subj' -f $statusdir/index.html -m text/html"); + + my $result=system("metasend -b -S 4000000 -t $emailto -s '$subj' -f $statusdir/index.html -m text/html >& /dev/null"); + if ($result != 0) { + $result=system("mail -s '$subj' $emailto < $statusdir/index.html"); + } } sub usage { @@ -577,18 +604,24 @@ Options: running, do not send emails do not check the lonc/d connections, do not generate lon-status + --justreload Only tell the daemons to reload the config files, + do not send emails do not + check if the daemons are running, do not + generate lon-status USAGE } # ================================================================ Main Program sub main () { - my ($oldlonc,$help,$justcheckdaemons,$noemail,$justcheckconnections); + my ($oldlonc,$help,$justcheckdaemons,$noemail,$justcheckconnections, + $justreload); &GetOptions("help" => \$help, "oldlonc" => \$oldlonc, "justcheckdaemons" => \$justcheckdaemons, "noemail" => \$noemail, - "justcheckconnections" => \$justcheckconnections + "justcheckconnections" => \$justcheckconnections, + "justreload" => \$justreload ); if ($help) { &usage(); return; } # --------------------------------- Read loncapa_apache.conf and loncapa.conf @@ -629,8 +662,9 @@ sub main () { my (%hostname,%hostdom,%hostrole,%spareid); while (my $configline=<$config>) { next if ($configline =~ /^(\#|\s*\$)/); - my ($id,$domain,$role,$name,$ip,$domdescr)=split(/:/,$configline); - if ($id && $domain && $role && $name && $ip) { + my ($id,$domain,$role,$name)=split(/:/,$configline); + if ($id && $domain && $role && $name) { + $name=~s/\s//g; $hostname{$id}=$name; $hostdom{$id}=$domain; $hostrole{$id}=$role; @@ -657,7 +691,7 @@ sub main () { my $fh; - if (!$justcheckdaemons && !$justcheckconnections) { + if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { $fh=&start_logging(\%hostdom,\%hostrole,\%hostname,\%spareid); &log_machine_info($fh); @@ -666,18 +700,29 @@ sub main () { &check_httpd_logs($fh); &rotate_lonnet_logs($fh); } - if (!$justcheckconnections) { + if (!$justcheckconnections && !$justreload) { &checkon_daemon($fh,'lonsql',200000); - &checkon_daemon($fh,'lond',40000,1); + if ( &checkon_daemon($fh,'lond',40000,'USR1') eq 'running') { + &checkon_daemon($fh,'lond',40000,'USR2'); + } my $args='new'; if ($oldlonc) { $args = ''; } - &checkon_daemon($fh,'lonc',40000,1,$args); + if ( &checkon_daemon($fh,'lonc',40000,'USR1',$args) eq 'running') { + &checkon_daemon($fh,'lonc',40000,'USR2',$args); + } &checkon_daemon($fh,'lonhttpd',40000); + &checkon_daemon($fh,'lonmemcached',40000); + } + if ($justreload) { + &checkon_daemon($fh,'lond',40000,'USR2'); + my $args='new'; + if ($oldlonc) { $args = ''; } + &checkon_daemon($fh,'lonc',40000,'USR2',$args); } - if (!$justcheckdaemons) { + if ($justcheckconnections) { &test_connections($fh,\%hostname); } - if (!$justcheckdaemons && !$justcheckconnections) { + if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { &check_delayed_msg($fh); &finish_logging($fh); &log_simplestatus();