version 1.53, 2004/06/09 13:30:41
|
version 1.66, 2006/02/06 09:08:56
|
Line 45 my $statusdir="/home/httpd/html/lon-stat
|
Line 45 my $statusdir="/home/httpd/html/lon-stat
|
|
|
# -------------------------------------------------- Non-critical communication |
# -------------------------------------------------- Non-critical communication |
sub reply { |
sub reply { |
my ($cmd,$server)=@_; |
my ($cmd,$server,$hostname)=@_; |
my $peerfile="$perlvar{'lonSockDir'}/$server"; |
my $peerfile="$perlvar{'lonSockDir'}/".$hostname->{$server}; |
my $client=IO::Socket::UNIX->new(Peer =>"$peerfile", |
my $client=IO::Socket::UNIX->new(Peer =>"$peerfile", |
Type => SOCK_STREAM, |
Type => SOCK_STREAM, |
Timeout => 10) |
Timeout => 10) |
or return "con_lost"; |
or return "con_lost"; |
print $client "$cmd\n"; |
print $client "sethost:$server:$cmd\n"; |
my $answer=<$client>; |
my $answer=<$client>; |
chomp($answer); |
chomp($answer); |
if (!$answer) { $answer="con_lost"; } |
if (!$answer) { $answer="con_lost"; } |
Line 92 sub start_daemon {
|
Line 92 sub start_daemon {
|
rename("$error_fname","$error_fname.1"); |
rename("$error_fname","$error_fname.1"); |
} |
} |
system("$perlvar{'lonDaemons'}/$progname 2>$perlvar{'lonDaemons'}/logs/${daemon}_errors"); |
system("$perlvar{'lonDaemons'}/$progname 2>$perlvar{'lonDaemons'}/logs/${daemon}_errors"); |
sleep 2; |
sleep 1; |
if (-e $pidfile) { |
if (-e $pidfile) { |
&log($fh,"<p>Seems like it started ...</p>"); |
&log($fh,"<p>Seems like it started ...</p>"); |
my $lfh=IO::File->new("$pidfile"); |
my $lfh=IO::File->new("$pidfile"); |
my $daemonpid=<$lfh>; |
my $daemonpid=<$lfh>; |
chomp($daemonpid); |
chomp($daemonpid); |
sleep 2; |
if ($daemonpid =~ /^\d+$/ && kill 0 => $daemonpid) { |
if (kill 0 => $daemonpid) { |
|
return 1; |
return 1; |
} else { |
} else { |
return 0; |
return 0; |
Line 111 sub start_daemon {
|
Line 110 sub start_daemon {
|
} |
} |
|
|
sub checkon_daemon { |
sub checkon_daemon { |
my ($fh,$daemon,$maxsize,$sendusr1,$args)=@_; |
my ($fh,$daemon,$maxsize,$send,$args)=@_; |
|
|
|
my $result; |
&log($fh,'<hr /><a name="'.$daemon.'" /><h2>'.$daemon.'</h2><h3>Log</h3><p style="white-space: pre;"><tt>'); |
&log($fh,'<hr /><a name="'.$daemon.'" /><h2>'.$daemon.'</h2><h3>Log</h3><p style="white-space: pre;"><tt>'); |
printf("%-10s ",$daemon); |
printf("%-15s ",$daemon); |
if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ |
if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ |
open (DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/$daemon.log|"); |
open (DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/$daemon.log|"); |
while (my $line=<DFH>) { |
while (my $line=<DFH>) { |
Line 135 sub checkon_daemon {
|
Line 135 sub checkon_daemon {
|
my $lfh=IO::File->new("$pidfile"); |
my $lfh=IO::File->new("$pidfile"); |
$daemonpid=<$lfh>; |
$daemonpid=<$lfh>; |
chomp($daemonpid); |
chomp($daemonpid); |
if (kill 0 => $daemonpid) { |
if ($daemonpid =~ /^\d+$/ && kill 0 => $daemonpid) { |
&log($fh,"<h3>$daemon at pid $daemonpid responding"); |
&log($fh,"<h3>$daemon at pid $daemonpid responding"); |
if ($sendusr1) { &log($fh,", sending USR1"); } |
if ($send) { &log($fh,", sending $send"); } |
&log($fh,"</h3>"); |
&log($fh,"</h3>"); |
if ($sendusr1) { kill USR1 => $daemonpid; } |
if ($send eq 'USR1') { kill USR1 => $daemonpid; } |
|
if ($send eq 'USR2') { kill USR2 => $daemonpid; } |
$restartflag=0; |
$restartflag=0; |
print "running\n"; |
if ($send eq 'USR2') { |
|
$result = 'reloaded'; |
|
print "reloaded\n"; |
|
} else { |
|
$result = 'running'; |
|
print "running\n"; |
|
} |
} else { |
} else { |
$errors++; |
$errors++; |
&log($fh,"<h3>$daemon at pid $daemonpid not responding</h3>"); |
&log($fh,"<h3>$daemon at pid $daemonpid not responding</h3>"); |
Line 152 sub checkon_daemon {
|
Line 159 sub checkon_daemon {
|
if ($restartflag==1) { |
if ($restartflag==1) { |
$simplestatus{$daemon}='off'; |
$simplestatus{$daemon}='off'; |
$errors++; |
$errors++; |
|
my $kadaemon=$daemon; |
|
if ($kadaemon eq 'lonmemcached') { $kadaemon='memcached'; } |
&log($fh,'<br><font color="red">Killall '.$daemon.': '. |
&log($fh,'<br><font color="red">Killall '.$daemon.': '. |
`killall $daemon 2>&1`.' - '); |
`killall $kadaemon 2>&1`.' - '); |
sleep 2; |
sleep 1; |
&log($fh,unlink($pidfile).' - '. |
&log($fh,unlink($pidfile).' - '. |
`killall -9 $daemon 2>&1`. |
`killall -9 $kadaemon 2>&1`. |
'</font><br>'); |
'</font><br>'); |
&log($fh,"<h3>$daemon not running, trying to start</h3>"); |
&log($fh,"<h3>$daemon not running, trying to start</h3>"); |
|
|
if (&start_daemon($fh,$daemon,$pidfile,$args)) { |
if (&start_daemon($fh,$daemon,$pidfile,$args)) { |
&log($fh,"<h3>$daemon at pid $daemonpid responding</h3>"); |
&log($fh,"<h3>$daemon at pid $daemonpid responding</h3>"); |
$simplestatus{$daemon}='restarted'; |
$simplestatus{$daemon}='restarted'; |
|
$result = 'started'; |
print "started\n"; |
print "started\n"; |
} else { |
} else { |
$errors++; |
$errors++; |
Line 172 sub checkon_daemon {
|
Line 182 sub checkon_daemon {
|
if (&start_daemon($fh,$daemon,$pidfile,$args)) { |
if (&start_daemon($fh,$daemon,$pidfile,$args)) { |
&log($fh,"<h3>$daemon at pid $daemonpid responding</h3>"); |
&log($fh,"<h3>$daemon at pid $daemonpid responding</h3>"); |
$simplestatus{$daemon}='restarted'; |
$simplestatus{$daemon}='restarted'; |
|
$result = 'started'; |
print "started\n"; |
print "started\n"; |
} else { |
} else { |
|
$result = 'failed'; |
print " failed\n"; |
print " failed\n"; |
$simplestatus{$daemon}='failed'; |
$simplestatus{$daemon}='failed'; |
$errors++; $errors++; |
$errors++; $errors++; |
Line 210 sub checkon_daemon {
|
Line 222 sub checkon_daemon {
|
} |
} |
|
|
&errout($fh); |
&errout($fh); |
|
return $result; |
} |
} |
|
|
# --------------------------------------------------------------------- Machine |
# --------------------------------------------------------------------- Machine |
Line 271 sub log_machine_info {
|
Line 284 sub log_machine_info {
|
if ($psproc>200) { $notices++; } |
if ($psproc>200) { $notices++; } |
if ($psproc>250) { $notices++; } |
if ($psproc>250) { $notices++; } |
|
|
|
&log($fh,"<h3>distprobe</h3>"); |
|
&log($fh,"<pre>"); |
|
open(DSH,"$perlvar{'lonDaemons'}/distprobe |"); |
|
while (my $line=<DSH>) { |
|
&log($fh,&encode_entities($line,'<>&"')); |
|
$psproc++; |
|
} |
|
close(DSH); |
|
&log($fh,"</pre>"); |
|
|
&errout($fh); |
&errout($fh); |
} |
} |
|
|
Line 463 sub test_connections {
|
Line 486 sub test_connections {
|
foreach my $tryserver (sort(keys(%{$hostname}))) { |
foreach my $tryserver (sort(keys(%{$hostname}))) { |
print("."); |
print("."); |
my $result; |
my $result; |
my $answer=reply("pong",$tryserver); |
my $answer=reply("ping",$tryserver,$hostname); |
if ($answer eq "$tryserver:$perlvar{'lonHostID'}") { |
if ($answer eq "$tryserver:$perlvar{'lonHostID'}") { |
$result="<b>ok</b>"; |
$result="<b>ok</b>"; |
$good++; |
$good++; |
Line 554 sub log_simplestatus {
|
Line 577 sub log_simplestatus {
|
sub send_mail { |
sub send_mail { |
print "sending mail\n"; |
print "sending mail\n"; |
my $emailto="$perlvar{'lonAdmEMail'}"; |
my $emailto="$perlvar{'lonAdmEMail'}"; |
if ($totalcount>1000) { |
if ($totalcount>2500) { |
$emailto.=",$perlvar{'lonSysEMail'}"; |
$emailto.=",$perlvar{'lonSysEMail'}"; |
} |
} |
my $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; |
my $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; |
|
|
my $result=system("metasend -b -t $emailto -s '$subj' -f $statusdir/index.html -m text/html >& /dev/null"); |
my $result=system("metasend -b -S 4000000 -t $emailto -s '$subj' -f $statusdir/index.html -m text/html >& /dev/null"); |
if ($result != 0) { |
if ($result != 0) { |
$result=system("mail -s '$subj' $emailto < $statusdir/index.html"); |
$result=system("mail -s '$subj' $emailto < $statusdir/index.html"); |
} |
} |
Line 581 Options:
|
Line 604 Options:
|
running, do not send emails do not |
running, do not send emails do not |
check the lonc/d connections, do not |
check the lonc/d connections, do not |
generate lon-status |
generate lon-status |
|
--justreload Only tell the daemons to reload the config files, |
|
do not send emails do not |
|
check if the daemons are running, do not |
|
generate lon-status |
|
|
USAGE |
USAGE |
} |
} |
|
|
# ================================================================ Main Program |
# ================================================================ Main Program |
sub main () { |
sub main () { |
my ($oldlonc,$help,$justcheckdaemons,$noemail,$justcheckconnections); |
my ($oldlonc,$help,$justcheckdaemons,$noemail,$justcheckconnections, |
|
$justreload); |
&GetOptions("help" => \$help, |
&GetOptions("help" => \$help, |
"oldlonc" => \$oldlonc, |
"oldlonc" => \$oldlonc, |
"justcheckdaemons" => \$justcheckdaemons, |
"justcheckdaemons" => \$justcheckdaemons, |
"noemail" => \$noemail, |
"noemail" => \$noemail, |
"justcheckconnections" => \$justcheckconnections |
"justcheckconnections" => \$justcheckconnections, |
|
"justreload" => \$justreload |
); |
); |
if ($help) { &usage(); return; } |
if ($help) { &usage(); return; } |
# --------------------------------- Read loncapa_apache.conf and loncapa.conf |
# --------------------------------- Read loncapa_apache.conf and loncapa.conf |
Line 633 sub main () {
|
Line 662 sub main () {
|
my (%hostname,%hostdom,%hostrole,%spareid); |
my (%hostname,%hostdom,%hostrole,%spareid); |
while (my $configline=<$config>) { |
while (my $configline=<$config>) { |
next if ($configline =~ /^(\#|\s*\$)/); |
next if ($configline =~ /^(\#|\s*\$)/); |
my ($id,$domain,$role,$name,$ip,$domdescr)=split(/:/,$configline); |
my ($id,$domain,$role,$name)=split(/:/,$configline); |
if ($id && $domain && $role && $name && $ip) { |
if ($id && $domain && $role && $name) { |
|
$name=~s/\s//g; |
$hostname{$id}=$name; |
$hostname{$id}=$name; |
$hostdom{$id}=$domain; |
$hostdom{$id}=$domain; |
$hostrole{$id}=$role; |
$hostrole{$id}=$role; |
Line 661 sub main () {
|
Line 691 sub main () {
|
|
|
|
|
my $fh; |
my $fh; |
if (!$justcheckdaemons && !$justcheckconnections) { |
if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { |
$fh=&start_logging(\%hostdom,\%hostrole,\%hostname,\%spareid); |
$fh=&start_logging(\%hostdom,\%hostrole,\%hostname,\%spareid); |
|
|
&log_machine_info($fh); |
&log_machine_info($fh); |
Line 670 sub main () {
|
Line 700 sub main () {
|
&check_httpd_logs($fh); |
&check_httpd_logs($fh); |
&rotate_lonnet_logs($fh); |
&rotate_lonnet_logs($fh); |
} |
} |
if (!$justcheckconnections) { |
if (!$justcheckconnections && !$justreload) { |
&checkon_daemon($fh,'lonsql',200000); |
&checkon_daemon($fh,'lonsql',200000); |
&checkon_daemon($fh,'lond',40000,1); |
if ( &checkon_daemon($fh,'lond',40000,'USR1') eq 'running') { |
|
&checkon_daemon($fh,'lond',40000,'USR2'); |
|
} |
my $args='new'; |
my $args='new'; |
if ($oldlonc) { $args = ''; } |
if ($oldlonc) { $args = ''; } |
&checkon_daemon($fh,'lonc',40000,1,$args); |
&checkon_daemon($fh,'lonc',40000,'USR2',$args); |
&checkon_daemon($fh,'lonhttpd',40000); |
&checkon_daemon($fh,'lonhttpd',40000); |
|
&checkon_daemon($fh,'lonmemcached',40000); |
|
} |
|
if ($justreload) { |
|
&checkon_daemon($fh,'lond',40000,'USR2'); |
|
my $args='new'; |
|
if ($oldlonc) { $args = ''; } |
|
&checkon_daemon($fh,'lonc',40000,'USR2',$args); |
} |
} |
if (!$justcheckdaemons) { |
if ($justcheckconnections) { |
&test_connections($fh,\%hostname); |
&test_connections($fh,\%hostname); |
} |
} |
if (!$justcheckdaemons && !$justcheckconnections) { |
if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { |
&check_delayed_msg($fh); |
&check_delayed_msg($fh); |
&finish_logging($fh); |
&finish_logging($fh); |
&log_simplestatus(); |
&log_simplestatus(); |