version 1.63, 2006/01/27 21:27:15
|
version 1.75, 2007/06/12 23:50:39
|
Line 32 use strict;
|
Line 32 use strict;
|
|
|
use lib '/home/httpd/lib/perl/'; |
use lib '/home/httpd/lib/perl/'; |
use LONCAPA::Configuration; |
use LONCAPA::Configuration; |
|
use Apache::lonnet; |
|
|
use IO::File; |
use IO::File; |
use IO::Socket; |
use IO::Socket; |
Line 43 use vars qw (%perlvar %simplestatus $err
|
Line 44 use vars qw (%perlvar %simplestatus $err
|
my $statusdir="/home/httpd/html/lon-status"; |
my $statusdir="/home/httpd/html/lon-status"; |
|
|
|
|
# -------------------------------------------------- Non-critical communication |
|
sub reply { |
|
my ($cmd,$server)=@_; |
|
my $peerfile="$perlvar{'lonSockDir'}/$server"; |
|
my $client=IO::Socket::UNIX->new(Peer =>"$peerfile", |
|
Type => SOCK_STREAM, |
|
Timeout => 10) |
|
or return "con_lost"; |
|
print $client "$cmd\n"; |
|
my $answer=<$client>; |
|
chomp($answer); |
|
if (!$answer) { $answer="con_lost"; } |
|
return $answer; |
|
} |
|
|
|
# --------------------------------------------------------- Output error status |
# --------------------------------------------------------- Output error status |
|
|
sub log { |
sub log { |
Line 76 sub errout {
|
Line 62 sub errout {
|
ENDERROUT |
ENDERROUT |
} |
} |
|
|
|
sub rotate_logfile { |
|
my ($file,$fh,$description) = @_; |
|
my $size=(stat($file))[7]; |
|
if ($size>40000) { |
|
&log($fh,"<p>Rotating $description ...</p>"); |
|
rename("$file.2","$file.3"); |
|
rename("$file.1","$file.2"); |
|
rename("$file","$file.1"); |
|
} |
|
} |
|
|
sub start_daemon { |
sub start_daemon { |
my ($fh,$daemon,$pidfile,$args) = @_; |
my ($fh,$daemon,$pidfile,$args) = @_; |
my $progname=$daemon; |
my $progname=$daemon; |
if ($daemon eq 'lonc' && $args eq 'new') { |
if ($daemon eq 'lonc') { |
$progname='loncnew'; |
$progname='loncnew'; |
print "new "; |
|
} |
} |
my $error_fname="$perlvar{'lonDaemons'}/logs/${daemon}_errors"; |
my $error_fname="$perlvar{'lonDaemons'}/logs/${daemon}_errors"; |
my $size=(stat($error_fname))[7]; |
&rotate_logfile($error_fname,$fh,'error logs'); |
if ($size>40000) { |
if ($daemon eq 'lonc') { |
&log($fh,"<p>Rotating error logs ...</p>"); |
&clean_sockets($fh); |
rename("$error_fname.2","$error_fname.3"); |
|
rename("$error_fname.1","$error_fname.2"); |
|
rename("$error_fname","$error_fname.1"); |
|
} |
} |
system("$perlvar{'lonDaemons'}/$progname 2>$perlvar{'lonDaemons'}/logs/${daemon}_errors"); |
system("$perlvar{'lonDaemons'}/$progname 2>$perlvar{'lonDaemons'}/logs/${daemon}_errors"); |
sleep 1; |
sleep 1; |
Line 208 sub checkon_daemon {
|
Line 201 sub checkon_daemon {
|
} |
} |
|
|
my $fname="$perlvar{'lonDaemons'}/logs/$daemon.log"; |
my $fname="$perlvar{'lonDaemons'}/logs/$daemon.log"; |
|
&rotate_logfile($fname,$fh,'logs'); |
my ($dev,$ino,$mode,$nlink, |
|
$uid,$gid,$rdev,$size, |
|
$atime,$mtime,$ctime, |
|
$blksize,$blocks)=stat($fname); |
|
|
|
if ($size>$maxsize) { |
|
&log($fh,"<p>Rotating logs ...</p>"); |
|
rename("$fname.2","$fname.3"); |
|
rename("$fname.1","$fname.2"); |
|
rename("$fname","$fname.1"); |
|
} |
|
|
|
&errout($fh); |
&errout($fh); |
return $result; |
return $result; |
Line 298 sub log_machine_info {
|
Line 280 sub log_machine_info {
|
} |
} |
|
|
sub start_logging { |
sub start_logging { |
my ($hostdom,$hostrole,$hostname,$spareid)=@_; |
|
my $fh=IO::File->new(">$statusdir/newstatus.html"); |
my $fh=IO::File->new(">$statusdir/newstatus.html"); |
my %simplestatus=(); |
my %simplestatus=(); |
my $now=time; |
my $now=time; |
Line 341 ENDHEADERS
|
Line 322 ENDHEADERS
|
&encode_entities($perlvar{$varname},'<>&"')."</td></tr>\n"); |
&encode_entities($perlvar{$varname},'<>&"')."</td></tr>\n"); |
} |
} |
&log($fh,"</table><h3>Hosts</h3><table border='2'>"); |
&log($fh,"</table><h3>Hosts</h3><table border='2'>"); |
foreach my $id (sort(keys(%{$hostname}))) { |
my %hostname = &Apache::lonnet::all_hostnames(); |
|
foreach my $id (sort(keys(%hostname))) { |
|
my $role = (&Apache::lonnet::is_library($id) ? 'library' |
|
: 'access'); |
&log($fh, |
&log($fh, |
"<tr><td>$id</td><td>".$hostdom->{$id}. |
"<tr><td>$id</td><td>".&Apache::lonnet::host_domain($id). |
"</td><td>".$hostrole->{$id}. |
"</td><td>".$role. |
"</td><td>".$hostname->{$id}."</td></tr>\n"); |
"</td><td>".&Apache::lonnet::hostname($id)."</td></tr>\n"); |
} |
} |
&log($fh,"</table><h3>Spare Hosts</h3><ol>"); |
&log($fh,"</table><h3>Spare Hosts</h3><ul>"); |
foreach my $id (sort(keys(%{$spareid}))) { |
foreach my $type (sort(keys(%Apache::lonnet::spareid))) { |
&log($fh,"<li>$id\n</li>"); |
&log($fh,"<li>$type\n<ol>"); |
|
foreach my $id (@{ $Apache::lonnet::spareid{$type} }) { |
|
&log($fh,"<li>$id</li>\n"); |
|
} |
|
&log($fh,"</ol>\n</li>\n"); |
} |
} |
&log($fh,"</ol>\n"); |
&log($fh,"</ul>\n"); |
return $fh; |
return $fh; |
} |
} |
|
|
Line 413 sub clean_lonIDs {
|
Line 401 sub clean_lonIDs {
|
&log($fh,"<h3>$active open session(s)</h3>"); |
&log($fh,"<h3>$active open session(s)</h3>"); |
} |
} |
|
|
|
# ----------------------------------------------------------- clean out sockets |
|
sub clean_sockets { |
|
my ($fh)=@_; |
|
my $cleaned=0; |
|
opendir(SOCKETS,$perlvar{'lonSockDir'}); |
|
while (my $fname=readdir(SOCKETS)) { |
|
next if (-d $fname |
|
|| $fname=~/(mysqlsock|maximasock|\Q$perlvar{'lonSockDir'}\E)/); |
|
$cleaned++; |
|
&log($fh,"Unlinking $fname<br />"); |
|
unlink("/home/httpd/sockets/$fname"); |
|
} |
|
&log($fh,"<p>Cleaned up ".$cleaned." stale sockets.</p>"); |
|
} |
|
|
|
|
# ----------------------------------------------------------------------- httpd |
# ----------------------------------------------------------------------- httpd |
sub check_httpd_logs { |
sub check_httpd_logs { |
Line 459 sub rotate_lonnet_logs {
|
Line 462 sub rotate_lonnet_logs {
|
} else { &log($fh,"No perm log\n") } |
} else { &log($fh,"No perm log\n") } |
|
|
my $fname="$perlvar{'lonDaemons'}/logs/lonnet.log"; |
my $fname="$perlvar{'lonDaemons'}/logs/lonnet.log"; |
|
&rotate_logfile($fname,$fh,'lonnet log'); |
my ($dev,$ino,$mode,$nlink, |
|
$uid,$gid,$rdev,$size, |
|
$atime,$mtime,$ctime, |
|
$blksize,$blocks)=stat($fname); |
|
|
|
if ($size>40000) { |
|
&log($fh,"<p>Rotating logs ...</p>"); |
|
rename("$fname.2","$fname.3"); |
|
rename("$fname.1","$fname.2"); |
|
rename("$fname","$fname.1"); |
|
} |
|
|
|
&log($fh,"</pre>"); |
&log($fh,"</pre>"); |
&errout($fh); |
&errout($fh); |
} |
} |
|
|
|
sub rotate_other_logs { |
|
my ($fh) = @_; |
|
my $fname="$perlvar{'lonDaemons'}/logs/autoenroll.log"; |
|
&rotate_logfile($fname,$fh,'Auto Enroll log'); |
|
$fname="$perlvar{'lonDaemons'}/logs/autocreate.log"; |
|
&rotate_logfile($fname,$fh,'Create Course log'); |
|
$fname="$perlvar{'lonDaemons'}/logs/searchcat.log"; |
|
&rotate_logfile($fname,$fh,'Search Cataloguing log'); |
|
} |
|
|
# ----------------------------------------------------------------- Connections |
# ----------------------------------------------------------------- Connections |
sub test_connections { |
sub test_connections { |
my ($fh,$hostname)=@_; |
my ($fh)=@_; |
&log($fh,'<hr /><a name="connections" /><h2>Connections</h2>'); |
&log($fh,'<hr /><a name="connections" /><h2>Connections</h2>'); |
print "testing connections\n"; |
print "testing connections\n"; |
&log($fh,"<table border='2'>"); |
&log($fh,"<table border='2'>"); |
my ($good,$bad)=(0,0); |
my ($good,$bad)=(0,0); |
foreach my $tryserver (sort(keys(%{$hostname}))) { |
my %hostname = &Apache::lonnet::all_hostnames(); |
|
foreach my $tryserver (sort(keys(%hostname))) { |
print("."); |
print("."); |
my $result; |
my $result; |
my $answer=reply("ping",$tryserver); |
my $answer=&Apache::lonnet::reply("ping",$tryserver); |
if ($answer eq "$tryserver:$perlvar{'lonHostID'}") { |
if ($answer eq "$tryserver:$perlvar{'lonHostID'}") { |
$result="<b>ok</b>"; |
$result="<b>ok</b>"; |
$good++; |
$good++; |
Line 536 sub check_delayed_msg {
|
Line 539 sub check_delayed_msg {
|
|
|
if ($unsend) { $simplestatus{'unsend'}=$unsend; } |
if ($unsend) { $simplestatus{'unsend'}=$unsend; } |
&log($fh,"<h3>Outgoing Buffer</h3>\n<pre>"); |
&log($fh,"<h3>Outgoing Buffer</h3>\n<pre>"); |
|
# list directory with delayed messages and remember offline servers |
|
my %servers=(); |
open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|"); |
open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|"); |
while (my $line=<DFH>) { |
while (my $line=<DFH>) { |
|
my ($server)=($line=~/\.(\w+)$/); |
|
if ($server) { $servers{$server}=1; } |
&log($fh,&encode_entities($line,'<>&"')); |
&log($fh,&encode_entities($line,'<>&"')); |
} |
} |
&log($fh,"</pre>\n"); |
&log($fh,"</pre>\n"); |
close (DFH); |
close (DFH); |
|
# pong to all servers that have delayed messages |
|
# this will trigger a reverse connection, which should flush the buffers |
|
foreach my $tryserver (keys %servers) { |
|
my $answer=&Apache::lonnet::reply("pong",$tryserver); |
|
&log($fh,"Pong to $tryserver: $answer<br />"); |
|
} |
} |
} |
|
|
sub finish_logging { |
sub finish_logging { |
Line 564 sub finish_logging {
|
Line 576 sub finish_logging {
|
} |
} |
|
|
sub log_simplestatus { |
sub log_simplestatus { |
rename ("$statusdir/newstatus.html","$statusdir/index.html"); |
rename("$statusdir/newstatus.html","$statusdir/index.html"); |
|
|
my $sfh=IO::File->new(">$statusdir/loncron_simple.txt"); |
my $sfh=IO::File->new(">$statusdir/loncron_simple.txt"); |
foreach (keys %simplestatus) { |
foreach (keys %simplestatus) { |
Line 593 sub usage {
|
Line 605 sub usage {
|
loncron - housekeeping program that checks up on various parts of Lon-CAPA |
loncron - housekeeping program that checks up on various parts of Lon-CAPA |
|
|
Options: |
Options: |
--help Display help |
--help Display |
--oldlonc When starting the lonc daemon use 'lonc' not 'loncnew' |
|
--noemail Do not send the status email |
--noemail Do not send the status email |
--justcheckconnections Only check the current status of the lonc/d |
--justcheckconnections Only check the current status of the lonc/d |
connections, do not send emails do not |
connections, do not send emails do not |
Line 614 USAGE
|
Line 625 USAGE
|
|
|
# ================================================================ Main Program |
# ================================================================ Main Program |
sub main () { |
sub main () { |
my ($oldlonc,$help,$justcheckdaemons,$noemail,$justcheckconnections, |
my ($help,$justcheckdaemons,$noemail,$justcheckconnections, |
$justreload); |
$justreload); |
&GetOptions("help" => \$help, |
&GetOptions("help" => \$help, |
"oldlonc" => \$oldlonc, |
|
"justcheckdaemons" => \$justcheckdaemons, |
"justcheckdaemons" => \$justcheckdaemons, |
"noemail" => \$noemail, |
"noemail" => \$noemail, |
"justcheckconnections" => \$justcheckconnections, |
"justcheckconnections" => \$justcheckconnections, |
Line 630 sub main () {
|
Line 640 sub main () {
|
undef $perlvarref; |
undef $perlvarref; |
delete $perlvar{'lonReceipt'}; # remove since sensitive and not needed |
delete $perlvar{'lonReceipt'}; # remove since sensitive and not needed |
delete $perlvar{'lonSqlAccess'}; # remove since sensitive and not needed |
delete $perlvar{'lonSqlAccess'}; # remove since sensitive and not needed |
|
chdir($perlvar{'lonDaemons'}); |
# --------------------------------------- Make sure that LON-CAPA is configured |
# --------------------------------------- Make sure that LON-CAPA is configured |
# I only test for one thing here (lonHostID). This is just a safeguard. |
# I only test for one thing here (lonHostID). This is just a safeguard. |
if ('{[[[[lonHostID]]]]}' eq $perlvar{'lonHostID'}) { |
if ('{[[[[lonHostID]]]]}' eq $perlvar{'lonHostID'}) { |
Line 656 sub main () {
|
Line 666 sub main () {
|
exit 1; |
exit 1; |
} |
} |
|
|
# ------------------------------------------------------------- Read hosts file |
# -------------------------------------------- Force reload of host information |
my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab"); |
&Apache::lonnet::load_hosts_tab(1); |
|
&Apache::lonnet::load_domain_tab(1); |
my (%hostname,%hostdom,%hostrole,%spareid); |
&Apache::lonnet::get_iphost(1); |
while (my $configline=<$config>) { |
|
next if ($configline =~ /^(\#|\s*\$)/); |
|
my ($id,$domain,$role,$name)=split(/:/,$configline); |
|
if ($id && $domain && $role && $name) { |
|
$name=~s/\s//g; |
|
$hostname{$id}=$name; |
|
$hostdom{$id}=$domain; |
|
$hostrole{$id}=$role; |
|
} |
|
} |
|
undef $config; |
|
|
|
# ------------------------------------------------------ Read spare server file |
|
$config=IO::File->new("$perlvar{'lonTabDir'}/spare.tab"); |
|
|
|
while (my $configline=<$config>) { |
|
chomp($configline); |
|
if (($configline) && ($configline ne $perlvar{'lonHostID'})) { |
|
$spareid{$configline}=1; |
|
} |
|
} |
|
undef $config; |
|
|
|
# ---------------------------------------------------------------- Start report |
# ---------------------------------------------------------------- Start report |
|
|
Line 692 sub main () {
|
Line 680 sub main () {
|
|
|
my $fh; |
my $fh; |
if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { |
if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { |
$fh=&start_logging(\%hostdom,\%hostrole,\%hostname,\%spareid); |
$fh=&start_logging(); |
|
|
&log_machine_info($fh); |
&log_machine_info($fh); |
&clean_tmp($fh); |
&clean_tmp($fh); |
&clean_lonIDs($fh); |
&clean_lonIDs($fh); |
&check_httpd_logs($fh); |
&check_httpd_logs($fh); |
&rotate_lonnet_logs($fh); |
&rotate_lonnet_logs($fh); |
|
&rotate_other_logs($fh); |
} |
} |
if (!$justcheckconnections && !$justreload) { |
if (!$justcheckconnections && !$justreload) { |
&checkon_daemon($fh,'lonsql',200000); |
&checkon_daemon($fh,'lonsql',200000); |
if ( &checkon_daemon($fh,'lond',40000,'USR1') eq 'running') { |
if ( &checkon_daemon($fh,'lond',40000,'USR1') eq 'running') { |
&checkon_daemon($fh,'lond',40000,'USR2'); |
&checkon_daemon($fh,'lond',40000,'USR2'); |
} |
} |
my $args='new'; |
&checkon_daemon($fh,'lonc',40000,'USR1'); |
if ($oldlonc) { $args = ''; } |
|
if ( &checkon_daemon($fh,'lonc',40000,'USR1',$args) eq 'running') { |
|
&checkon_daemon($fh,'lond',40000,'USR2',$args); |
|
} |
|
&checkon_daemon($fh,'lonhttpd',40000); |
&checkon_daemon($fh,'lonhttpd',40000); |
&checkon_daemon($fh,'lonmemcached',40000); |
&checkon_daemon($fh,'lonmemcached',40000); |
|
&checkon_daemon($fh,'lonmaxima',40000); |
} |
} |
if ($justreload) { |
if ($justreload) { |
&checkon_daemon($fh,'lond',40000,'USR2'); |
&checkon_daemon($fh,'lond',40000,'USR2'); |
my $args='new'; |
&checkon_daemon($fh,'lonc',40000,'USR2'); |
if ($oldlonc) { $args = ''; } |
|
&checkon_daemon($fh,'lonc',40000,'USR2',$args); |
|
} |
} |
if ($justcheckconnections) { |
if ($justcheckconnections) { |
&test_connections($fh,\%hostname); |
&test_connections($fh); |
} |
} |
if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { |
if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { |
&check_delayed_msg($fh); |
&check_delayed_msg($fh); |