--- loncom/loncron 2004/05/11 19:12:50 1.46
+++ loncom/loncron 2009/06/11 00:15:27 1.81
@@ -1,47 +1,50 @@
#!/usr/bin/perl
-# The LearningOnline Network
-# Housekeeping program, started by cron
+# Housekeeping program, started by cron, loncontrol and loncron.pl
#
-# (TCP networking package
-# 6/1/99,6/2,6/10,6/11,6/12,6/14,6/26,6/28,6/29,6/30,
-# 7/1,7/2,7/9,7/10,7/12 Gerd Kortemeyer)
+# $Id: loncron,v 1.81 2009/06/11 00:15:27 raeburn Exp $
+#
+# Copyright Michigan State University Board of Trustees
+#
+# This file is part of the LearningOnline Network with CAPA (LON-CAPA).
+#
+# LON-CAPA is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# LON-CAPA is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with LON-CAPA; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+#
+# /home/httpd/html/adm/gpl.txt
+#
+# http://www.lon-capa.org/
#
-# 7/14,7/15,7/19,7/21,7/22,11/18,
-# 2/8 Gerd Kortemeyer
-# 12/23 Gerd Kortemeyer
-# YEAR=2001
-# 09/04,09/06,11/26 Gerd Kortemeyer
$|=1;
+use strict;
use lib '/home/httpd/lib/perl/';
use LONCAPA::Configuration;
+use Apache::lonnet;
+use Apache::loncommon;
use IO::File;
use IO::Socket;
-use strict;
+use HTML::Entities;
+use Getopt::Long;
#globals
use vars qw (%perlvar %simplestatus $errors $warnings $notices $totalcount);
my $statusdir="/home/httpd/html/lon-status";
-# -------------------------------------------------- Non-critical communication
-sub reply {
- my ($cmd,$server)=@_;
- my $peerfile="$perlvar{'lonSockDir'}/$server";
- my $client=IO::Socket::UNIX->new(Peer =>"$peerfile",
- Type => SOCK_STREAM,
- Timeout => 10)
- or return "con_lost";
- print $client "$cmd\n";
- my $answer=<$client>;
- chomp($answer);
- if (!$answer) { $answer="con_lost"; }
- return $answer;
-}
-
# --------------------------------------------------------- Output error status
sub log {
@@ -52,45 +55,60 @@ sub log {
sub errout {
my $fh=shift;
&log($fh,(<
+ Rotating $description ... ");
+ &log($fh," Seems like it started ... ");
+ &log($fh," Seems like that did not work! ');
+ printf("%-15s ",$daemon);
if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){
open (DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/$daemon.log|");
while (my $line=
+
ENDERROUT
}
+sub rotate_logfile {
+ my ($file,$fh,$description) = @_;
+ my $size=(stat($file))[7];
+ if ($size>40000) {
+ &log($fh,"
Notices $notices Warnings $warnings
- Errors $errors '.$daemon.'
Log
');
- printf("%-10s ",$daemon);
+ my $result;
+ &log($fh,'
");
+ &log($fh,"'.$daemon.'
Log
"); + &log($fh,"
Give it one more try ...
"); print " "; - if (&start_daemon($fh,$daemon,$pidfile)) { + if (&start_daemon($fh,$daemon,$pidfile,$args)) { &log($fh,""); + &log($fh,"
Unable to start $daemon
"); } } @@ -167,31 +197,21 @@ sub checkon_daemon { if ($line=~/CRITICAL/) { $notices++; } }; close (DFH); - &log($fh,""); + &log($fh,""); } } my $fname="$perlvar{'lonDaemons'}/logs/$daemon.log"; - - my ($dev,$ino,$mode,$nlink, - $uid,$gid,$rdev,$size, - $atime,$mtime,$ctime, - $blksize,$blocks)=stat($fname); - - if ($size>$maxsize) { - &log($fh,"Rotating logs ..."); - rename("$fname.2","$fname.3"); - rename("$fname.1","$fname.2"); - rename("$fname","$fname.1"); - } + &rotate_logfile($fname,$fh,'logs'); &errout($fh); + return $result; } # --------------------------------------------------------------------- Machine sub log_machine_info { my ($fh)=@_; - &log($fh,'
"); my $psproc=0; - open (PSH,"ps -aux --cols 140 |"); + open (PSH,"ps aux --cols 140 |"); while (my $line=) { - &log($fh,"$line"); + &log($fh,&encode_entities($line,'<>&"')); $psproc++; } close (PSH); @@ -247,11 +267,20 @@ sub log_machine_info { if ($psproc>200) { $notices++; } if ($psproc>250) { $notices++; } + &log($fh," distprobe
"); + &log($fh,""); + open(DSH,"$perlvar{'lonDaemons'}/distprobe |"); + while (my $line="); + &errout($fh); } sub start_logging { - my ($hostdom,$hostrole,$hostname,$spareid)=@_; my $fh=IO::File->new(">$statusdir/newstatus.html"); my %simplestatus=(); my $now=time; @@ -264,53 +293,60 @@ sub start_logging {) { + &log($fh,&encode_entities($line,'<>&"')); + $psproc++; + } + close(DSH); + &log($fh," LON Status Report $perlvar{'lonHostID'} - +LON Status Report $perlvar{'lonHostID'}
$date ($now)
-
-- Configuration -
- Machine Information -
- Temporary Files -
- Session Tokens -
- httpd -
- lonsql -
- lond -
- lonc -
- lonhttpd -
- lonnet -
- Connections -
- Delayed Messages -
- Error Coindex.html.unt +
- Configuration
+- Machine Information
+- Temporary Files
+- Session Tokens
+- httpd
+- lonsql
+- lond
+- lonc
+- lonnet
+- Connections
+- Delayed Messages
+- Error Count
- +
+Configuration
PerlVars
-+
ENDHEADERS foreach my $varname (sort(keys(%perlvar))) { - &log($fh,"
\n"); + &log($fh," $varname $perlvar{$varname} \n"); } - &log($fh," $varname ". + &encode_entities($perlvar{$varname},'<>&"')." Hosts
"); - foreach my $id (sort(keys(%{$hostname}))) { + &log($fh,"
Hosts
"); + my %hostname = &Apache::lonnet::all_hostnames(); + foreach my $id (sort(keys(%hostname))) { + my $role = (&Apache::lonnet::is_library($id) ? 'library' + : 'access'); &log($fh, - "
\n"); - } - &log($fh," $id ".$hostdom->{$id}. - " ".$hostrole->{$id}. - " ".$hostname->{$id}." Spare Hosts
"); - foreach my $id (sort(keys(%{$spareid}))) { - &log($fh,"
- $id\n"); + "
\n"); + } + &log($fh," $id ".&Apache::lonnet::host_domain($id). + " ".$role. + " ".&Apache::lonnet::hostname($id)." Spare Hosts
"); + foreach my $type (sort(keys(%Apache::lonnet::spareid))) { + &log($fh,"
\n"); return $fh; } # --------------------------------------------------------------- clean out tmp sub clean_tmp { my ($fh)=@_; - &log($fh,'- $type\n
\n"); } - &log($fh,"\n"); + &log($fh,""); + foreach my $id (@{ $Apache::lonnet::spareid{$type} }) { + &log($fh,"
\n- $id
\n"); + } + &log($fh,"Temporary Files
'); + &log($fh,'Temporary Files
'); my $cleaned=0; my $old=0; while (my $fname=<$perlvar{'lonDaemons'}/tmp/*>) { @@ -343,7 +379,7 @@ sub clean_tmp { # ------------------------------------------------------------ clean out lonIDs sub clean_lonIDs { my ($fh)=@_; - &log($fh,'Session Tokens
'); + &log($fh,'Session Tokens
'); my $cleaned=0; my $active=0; while (my $fname=<$perlvar{'lonIDsDir'}/*>) { @@ -361,18 +397,33 @@ sub clean_lonIDs { $active++; } } - &log($fh,"Cleaned up ".$cleaned." stale session token(s)."); + &log($fh,"
Cleaned up ".$cleaned." stale session token(s).
"); &log($fh,"$active open session(s)
"); } +# ----------------------------------------------------------- clean out sockets +sub clean_sockets { + my ($fh)=@_; + my $cleaned=0; + opendir(SOCKETS,$perlvar{'lonSockDir'}); + while (my $fname=readdir(SOCKETS)) { + next if (-d $fname + || $fname=~/(mysqlsock|maximasock|rsock|\Q$perlvar{'lonSockDir'}\E)/); + $cleaned++; + &log($fh,"Unlinking $fname
"); + unlink("/home/httpd/sockets/$fname"); + } + &log($fh,"Cleaned up ".$cleaned." stale sockets.
"); +} + # ----------------------------------------------------------------------- httpd sub check_httpd_logs { my ($fh)=@_; - &log($fh,'httpd
Access Log
'); + &log($fh,'httpd
Access Log
'); open (DFH,"tail -n25 /etc/httpd/logs/access_log|"); - while (my $line=) { &log($fh,"$line") }; + while (my $line= ) { &log($fh,&encode_entities($line,'<>&"')) }; close (DFH); &log($fh," Error Log
"); @@ -389,14 +440,14 @@ sub check_httpd_logs { # ---------------------------------------------------------------------- lonnet -sub rotate_logs { +sub rotate_lonnet_logs { my ($fh)=@_; - &log($fh,'lonnet
Temp Log
'); + &log($fh,'lonnet
Temp Log
'); print "checking logs\n"; if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){ open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|"); while (my $line="); &errout($fh); } +sub rotate_other_logs { + my ($fh) = @_; + my $fname="$perlvar{'lonDaemons'}/logs/autoenroll.log"; + &rotate_logfile($fname,$fh,'Auto Enroll log'); + $fname="$perlvar{'lonDaemons'}/logs/autocreate.log"; + &rotate_logfile($fname,$fh,'Create Course log'); + $fname="$perlvar{'lonDaemons'}/logs/searchcat.log"; + &rotate_logfile($fname,$fh,'Search Cataloguing log'); +} + # ----------------------------------------------------------------- Connections sub test_connections { - my ($fh,$hostname)=@_; - &log($fh,') { - &log($fh,"$line"); + &log($fh,&encode_entities($line,'<>&"')); } close (DFH); } @@ -405,51 +456,58 @@ sub rotate_logs { if (-e "$perlvar{'lonDaemons'}/logs/lonnet.perm.log") { open(DFH,"tail -n10 $perlvar{'lonDaemons'}/logs/lonnet.perm.log|"); while (my $line= ) { - &log($fh,"$line"); + &log($fh,&encode_entities($line,'<>&"')); } close (DFH); } else { &log($fh,"No perm log\n") } my $fname="$perlvar{'lonDaemons'}/logs/lonnet.log"; - - my ($dev,$ino,$mode,$nlink, - $uid,$gid,$rdev,$size, - $atime,$mtime,$ctime, - $blksize,$blocks)=stat($fname); - - if ($size>40000) { - &log($fh,"Rotating logs ... "); - rename("$fname.2","$fname.3"); - rename("$fname.1","$fname.2"); - rename("$fname","$fname.1"); - } + &rotate_logfile($fname,$fh,'lonnet log'); &log($fh,"
Connections
'); + my ($fh)=@_; + &log($fh,'Connections
'); print "testing connections\n"; - &log($fh,""); - foreach my $tryserver (sort(keys(%{$hostname}))) { + &log($fh,"
"); + my ($good,$bad)=(0,0); + my %hostname = &Apache::lonnet::all_hostnames(); + foreach my $tryserver (sort(keys(%hostname))) { print("."); my $result; - my $answer=reply("pong",$tryserver); + my $answer=&Apache::lonnet::reply("ping",$tryserver); if ($answer eq "$tryserver:$perlvar{'lonHostID'}") { $result="ok"; + $good++; } else { $result=$answer; $warnings++; - if ($answer eq 'con_lost') { $warnings++; } + if ($answer eq 'con_lost') { + $bad++; + $warnings++; + } else { + $good++; #self connection + } } if ($answer =~ /con_lost/) { print(" $tryserver down\n"); } &log($fh,"
"); - + print "\n$good good, $bad bad connections\n"; &errout($fh); } @@ -457,7 +515,7 @@ sub test_connections { # ------------------------------------------------------------ Delayed messages sub check_delayed_msg { my ($fh)=@_; - &log($fh,'\n"); } &log($fh," $tryserver $result Delayed Messages
'); + &log($fh,'Delayed Messages
'); print "checking buffers\n"; &log($fh,'Scanning Permanent Log
'); @@ -476,28 +534,38 @@ sub check_delayed_msg { if ($sdf eq 'D') { $unsend++; } } - &log($fh,"Total unsend messages: $unsend\n"); + &log($fh,"
Total unsend messages: $unsend
\n"); $warnings=$warnings+5*$unsend; if ($unsend) { $simplestatus{'unsend'}=$unsend; } - &log($fh,"Outgoing Buffer
"); - + &log($fh,"Outgoing Buffer
\n"); +# list directory with delayed messages and remember offline servers + my %servers=(); open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|"); - while (my $line=\n"); close (DFH); +# pong to all servers that have delayed messages +# this will trigger a reverse connection, which should flush the buffers + foreach my $tryserver (keys %servers) { + my $answer=&Apache::lonnet::reply("pong",$tryserver); + &log($fh,"Pong to $tryserver: $answer) { - &log($fh,"$line
"); + while (my $line=) { + my ($server)=($line=~/\.(\w+)$/); + if ($server) { $servers{$server}=1; } + &log($fh,&encode_entities($line,'<>&"')); } + &log($fh,"
"); + } } sub finish_logging { my ($fh)=@_; - &log($fh,"\n"); + &log($fh,"\n"); $totalcount=$notices+4*$warnings+100*$errors; &errout($fh); &log($fh,"Total Error Count: $totalcount
"); my $now=time; my $date=localtime($now); - &log($fh,"
$date ($now)