;
+ close(PROBE);
+ if ($line=~/^CHECKOUTTOKEN\&/) {
+ if ($since>365*$perlvar{'lonExpire'}) {
+ if (unlink($fname)) {
+ $cleaned++;
+ } elsif (ref($errors->{file}) eq 'ARRAY') {
+ push(@{$errors->{file}},$fname);
+ }
+ } else {
+ $old++;
+ }
+ } else {
+ if (unlink($fname)) {
+ $cleaned++;
+ } elsif (ref($errors->{file}) eq 'ARRAY') {
+ push(@{$errors->{file}},$fname);
+ }
+ }
+ } elsif (ref($errors->{failopen}) eq 'ARRAY') {
+ push(@{$errors->{failopen}},$fname);
+ }
+ } else {
+ if (unlink($fname)) {
+ $cleaned++;
+ } elsif (ref($errors->{file}) eq 'ARRAY') {
+ push(@{$errors->{file}},$fname);
+ }
+ }
+ }
+ }
+ }
+ closedir($dh);
+ } elsif (ref($errors->{failopen}) eq 'ARRAY') {
+ push(@{$errors->{failopen}},$path);
}
- &log($fh,"Cleaned up ".$cleaned." files (".$old." old checkout tokens).");
+ return ($cleaned,$old,$removed);
}
# ------------------------------------------------------------ clean out lonIDs
@@ -391,7 +496,7 @@ sub clean_lonIDs {
my $since=$now-$mtime;
if ($since>$perlvar{'lonExpire'}) {
$cleaned++;
- &log($fh,"Unlinking $fname
");
+ &log($fh,"Unlinking $fname
");
unlink("$fname");
} else {
$active++;
@@ -401,6 +506,60 @@ sub clean_lonIDs {
&log($fh,"$active open session(s)
");
}
+# -------------------------------------------------------- clean out balanceIDs
+
+sub clean_balanceIDs {
+ my ($fh)=@_;
+ &log($fh,'
Session Tokens
');
+ my $cleaned=0;
+ my $active=0;
+ if (-d $perlvar{'lonBalanceDir'}) {
+ while (my $fname=<$perlvar{'balanceDir'}/*.id>) {
+ my ($dev,$ino,$mode,$nlink,
+ $uid,$gid,$rdev,$size,
+ $atime,$mtime,$ctime,
+ $blksize,$blocks)=stat($fname);
+ my $now=time;
+ my $since=$now-$mtime;
+ if ($since>$perlvar{'lonExpire'}) {
+ $cleaned++;
+ &log($fh,"Unlinking $fname
");
+ unlink("$fname");
+ } else {
+ $active++;
+ }
+ }
+ }
+ &log($fh,"Cleaned up ".$cleaned." stale balancer files
");
+ &log($fh,"$active unexpired balancer files
");
+}
+
+# ------------------------------------------------ clean out webDAV Session IDs
+sub clean_webDAV_sessionIDs {
+ my ($fh)=@_;
+ if ($perlvar{'lonRole'} eq 'library') {
+ &log($fh,'
WebDAV Session Tokens
');
+ my $cleaned=0;
+ my $active=0;
+ my $now = time;
+ if (-d $perlvar{'lonDAVsessDir'}) {
+ while (my $fname=<$perlvar{'lonDAVsessDir'}/*>) {
+ my @stats = stat($fname);
+ my $since=$now-$stats[9];
+ if ($since>$perlvar{'lonExpire'}) {
+ $cleaned++;
+ &log($fh,"Unlinking $fname
");
+ unlink("$fname");
+ } else {
+ $active++;
+ }
+ }
+ &log($fh,"Cleaned up ".$cleaned." stale webDAV session token(s).
");
+ &log($fh,"$active open webDAV session(s)
");
+ }
+ }
+}
+
# ----------------------------------------------------------- clean out sockets
sub clean_sockets {
my ($fh)=@_;
@@ -420,21 +579,13 @@ sub clean_sockets {
# ----------------------------------------------------------------------- httpd
sub check_httpd_logs {
my ($fh)=@_;
- &log($fh,'
httpd
Access Log
');
-
- open (DFH,"tail -n25 /etc/httpd/logs/access_log|");
- while (my $line=) { &log($fh,&encode_entities($line,'<>&"')) };
- close (DFH);
-
- &log($fh,"
Error Log
");
-
- open (DFH,"tail -n25 /etc/httpd/logs/error_log|");
- while (my $line=) {
- &log($fh,"$line");
- if ($line=~/\[error\]/) { $notices++; }
+ if (open(PIPE,"./lchttpdlogs|")) {
+ while (my $line=) {
+ &log($fh,$line);
+ if ($line=~/\[error\]/) { $notices++; }
+ }
+ close(PIPE);
}
- close (DFH);
- &log($fh,"
");
&errout($fh);
}
@@ -443,7 +594,7 @@ sub check_httpd_logs {
sub rotate_lonnet_logs {
my ($fh)=@_;
&log($fh,'
lonnet
Temp Log
');
- print "checking logs\n";
+ print "Checking logs.\n";
if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
while (my $line=) {
@@ -470,19 +621,24 @@ sub rotate_lonnet_logs {
sub rotate_other_logs {
my ($fh) = @_;
- my $fname="$perlvar{'lonDaemons'}/logs/autoenroll.log";
- &rotate_logfile($fname,$fh,'Auto Enroll log');
- $fname="$perlvar{'lonDaemons'}/logs/autocreate.log";
- &rotate_logfile($fname,$fh,'Create Course log');
- $fname="$perlvar{'lonDaemons'}/logs/searchcat.log";
- &rotate_logfile($fname,$fh,'Search Cataloguing log');
+ my %logs = (
+ autoenroll => 'Auto Enroll log',
+ autocreate => 'Create Course log',
+ searchcat => 'Search Cataloguing log',
+ autoupdate => 'Auto Update log',
+ refreshcourseids_db => 'Refresh CourseIDs db log',
+ );
+ foreach my $item (keys(%logs)) {
+ my $fname=$perlvar{'lonDaemons'}.'/logs/'.$item.'.log';
+ &rotate_logfile($fname,$fh,$logs{$item});
+ }
}
# ----------------------------------------------------------------- Connections
sub test_connections {
my ($fh)=@_;
&log($fh,'
Connections
');
- print "testing connections\n";
+ print "Testing connections.\n";
&log($fh,"");
my ($good,$bad)=(0,0);
my %hostname = &Apache::lonnet::all_hostnames();
@@ -516,18 +672,24 @@ sub test_connections {
sub check_delayed_msg {
my ($fh)=@_;
&log($fh,'
Delayed Messages
');
- print "checking buffers\n";
+ print "Checking buffers.\n";
&log($fh,'Scanning Permanent Log
');
my $unsend=0;
+ my %hostname = &Apache::lonnet::all_hostnames();
+ my $numhosts = scalar(keys(%hostname));
+
my $dfh=IO::File->new("$perlvar{'lonDaemons'}/logs/lonnet.perm.log");
while (my $line=<$dfh>) {
my ($time,$sdf,$dserv,$dcmd)=split(/:/,$line);
+ if ($numhosts) {
+ next unless ($hostname{$dserv});
+ }
if ($sdf eq 'F') {
my $local=localtime($time);
- &log($fh,"Failed: $time, $dserv, $dcmd
");
+ &log($fh,"Failed: $time, $dserv, $dcmd
");
$warnings++;
}
if ($sdf eq 'S') { $unsend--; }
@@ -535,7 +697,9 @@ sub check_delayed_msg {
}
&log($fh,"Total unsend messages: $unsend
\n");
- $warnings=$warnings+5*$unsend;
+ if ($unsend > 0) {
+ $warnings=$warnings+5*$unsend;
+ }
if ($unsend) { $simplestatus{'unsend'}=$unsend; }
&log($fh,"Outgoing Buffer
\n");
@@ -551,9 +715,24 @@ sub check_delayed_msg {
close (DFH);
# pong to all servers that have delayed messages
# this will trigger a reverse connection, which should flush the buffers
- foreach my $tryserver (keys %servers) {
- my $answer=&Apache::lonnet::reply("pong",$tryserver);
- &log($fh,"Pong to $tryserver: $answer
");
+ foreach my $tryserver (sort(keys(%servers))) {
+ if ($hostname{$tryserver} || !$numhosts) {
+ my $answer;
+ eval {
+ local $SIG{ ALRM } = sub { die "TIMEOUT" };
+ alarm(20);
+ $answer = &Apache::lonnet::reply("pong",$tryserver);
+ alarm(0);
+ };
+ if ($@ && $@ =~ m/TIMEOUT/) {
+ &log($fh,"Attempted pong to $tryserver timed out
");
+ print "Time out while contacting: $tryserver for pong.\n";
+ } else {
+ &log($fh,"Pong to $tryserver: $answer
");
+ }
+ } else {
+ &log($fh,"$tryserver has delayed messages, but is not part of the cluster -- skipping 'Pong'.
");
+ }
}
}
@@ -566,7 +745,7 @@ sub finish_logging {
my $now=time;
my $date=localtime($now);
&log($fh,"
$date ($now)\n");
- print "lon-status webpage updated\n";
+ print "lon-status webpage updated.\n";
$fh->close();
if ($errors) { $simplestatus{'errors'}=$errors; }
@@ -586,8 +765,183 @@ sub log_simplestatus {
$sfh->close();
}
+sub write_loncaparevs {
+ print "Retrieving LON-CAPA version information.\n";
+ my %hostname = &Apache::lonnet::all_hostnames();
+ my $output;
+ foreach my $id (sort(keys(%hostname))) {
+ if ($id ne '') {
+ my $loncaparev;
+ eval {
+ local $SIG{ ALRM } = sub { die "TIMEOUT" };
+ alarm(10);
+ $loncaparev =
+ &Apache::lonnet::get_server_loncaparev('',$id,1,'loncron');
+ alarm(0);
+ };
+ if ($@ && $@ =~ m/TIMEOUT/) {
+ print "Time out while contacting lonHost: $id for version.\n";
+ }
+ if ($loncaparev =~ /^[\w.\-]+$/) {
+ $output .= $id.':'.$loncaparev."\n";
+ }
+ }
+ }
+ if ($output) {
+ if (open(my $fh,">$perlvar{'lonTabDir'}/loncaparevs.tab")) {
+ print $fh $output;
+ close($fh);
+ &Apache::lonnet::load_loncaparevs();
+ }
+ }
+ return;
+}
+
+sub write_serverhomeIDs {
+ print "Retrieving LON-CAPA lonHostID information.\n";
+ my %name_to_host = &Apache::lonnet::all_names();
+ my $output;
+ foreach my $name (sort(keys(%name_to_host))) {
+ if ($name ne '') {
+ if (ref($name_to_host{$name}) eq 'ARRAY') {
+ my $serverhomeID;
+ eval {
+ local $SIG{ ALRM } = sub { die "TIMEOUT" };
+ alarm(10);
+ $serverhomeID =
+ &Apache::lonnet::get_server_homeID($name,1,'loncron');
+ alarm(0);
+ };
+ if ($@ && $@ =~ m/TIMEOUT/) {
+ print "Time out while contacting server: $name\n";
+ }
+ if ($serverhomeID ne '') {
+ $output .= $name.':'.$serverhomeID."\n";
+ } else {
+ $output .= $name.':'.$name_to_host{$name}->[0]."\n";
+ }
+ }
+ }
+ }
+ if ($output) {
+ if (open(my $fh,">$perlvar{'lonTabDir'}/serverhomeIDs.tab")) {
+ print $fh $output;
+ close($fh);
+ &Apache::lonnet::load_serverhomeIDs();
+ }
+ }
+ return;
+}
+
+sub write_checksums {
+ my $distro = &LONCAPA::distro();
+ if ($distro) {
+ print "Retrieving file version and checksumming.\n";
+ my $numchksums = 0;
+ my ($chksumsref,$versionsref) =
+ &LONCAPA::Checksumming::get_checksums($distro,$perlvar{'lonDaemons'},
+ $perlvar{'lonLib'},
+ $perlvar{'lonIncludes'},
+ $perlvar{'lonTabDir'});
+ if (ref($chksumsref) eq 'HASH') {
+ $numchksums = scalar(keys(%{$chksumsref}));
+ }
+ print "File version retrieved and checksumming completed for $numchksums files.\n";
+ } else {
+ print "File version retrieval and checksumming skipped - could not determine Linux distro.\n";
+ }
+ return;
+}
+
+sub write_hostips {
+ my $lontabdir = $perlvar{'lonTabDir'};
+ my $defdom = $perlvar{'lonDefDomain'};
+ my $lonhost = $perlvar{'lonHostID'};
+ my $newfile = "$lontabdir/currhostips.tab";
+ my $oldfile = "$lontabdir/prevhostips.tab";
+ my (%prevhosts,%currhosts,%ipchange);
+ if ((-e $newfile) && (-s $newfile)) {
+ move($newfile,$oldfile);
+ chmod(0644,$oldfile);
+ if (open(my $fh,'<',$oldfile)) {
+ while (my $line=<$fh>) {
+ chomp($line);
+ if ($line =~ /^([^:]+):([\d.]+)$/) {
+ $prevhosts{$1} = $2;
+ }
+ }
+ close($fh);
+ }
+ }
+ my ($ip_info,$cached) =
+ &Apache::lonnet::is_cached_new('iphost','iphost');
+ if (!$cached) {
+ &Apache::lonnet::get_iphost();
+ ($ip_info,$cached) =
+ &Apache::lonnet::is_cached_new('iphost','iphost');
+ }
+ if (ref($ip_info) eq 'ARRAY') {
+ %currhosts = %{$ip_info->[1]};
+ if (open(my $fh,'>',$newfile)) {
+ foreach my $key (keys(%currhosts)) {
+ print $fh "$key:$currhosts{$key}\n";
+ }
+ close($fh);
+ chmod(0644,$newfile);
+ }
+ }
+ if (keys(%prevhosts) && keys(%currhosts)) {
+ foreach my $key (keys(%prevhosts)) {
+ unless ($currhosts{$key} eq $prevhosts{$key}) {
+ $ipchange{$key} = $prevhosts{$key}.'|'.$currhosts{$key};
+ }
+ }
+ foreach my $key (keys(%currhosts)) {
+ unless ($currhosts{$key} eq $prevhosts{$key}) {
+ $ipchange{$key} = $prevhosts{$key}.' | '.$currhosts{$key};
+ }
+ }
+ }
+ if (&Apache::lonnet::domain($defdom,'primary') eq $lonhost) {
+ if (keys(%ipchange)) {
+ if (open(my $fh,'>>',$perlvar{'lonDaemons'}.'/logs/hostip.log')) {
+ print $fh "********************\n".localtime(time).' Changes --'."\n".
+ "Hostname | Previous IP | New IP\n".
+ "--------------------------------\n";
+ foreach my $hostname (sort(keys(%ipchange))) {
+ print $fh "$hostname | $ipchange{$hostname}\n";
+ }
+ print $fh "\n*******************\n\n";
+ close($fh);
+ }
+ my $emailto = &Apache::loncommon::build_recipient_list(undef,
+ 'hostipmail',$defdom);
+ if ($emailto) {
+ my $subject = "LON-CAPA Hostname to IP change ($perlvar{'lonHostID'})";
+ my $chgmail = "To: $emailto\n".
+ "Subject: $subject\n".
+ "Content-type: text/plain\; charset=UTF-8\n".
+ "MIME-Version: 1.0\n\n".
+ "Host/IP changes\n".
+ " \n".
+ "Hostname | Previous IP | New IP\n".
+ "--------------------------------\n";
+ foreach my $hostname (sort(keys(%ipchange))) {
+ $chgmail .= "$hostname | $ipchange{$hostname}\n";
+ }
+ $chgmail .= "\n\n";
+ if (open(my $mailh, "|/usr/lib/sendmail -oi -t -odb")) {
+ print $mailh $chgmail;
+ close($mailh);
+ print "Sending mail notification of hostname/IP changes.\n";
+ }
+ }
+ }
+ }
+ return;
+}
+
sub send_mail {
- print "sending mail\n";
my $defdom = $perlvar{'lonDefDomain'};
my $origmail = $perlvar{'lonAdmEMail'};
my $emailto = &Apache::loncommon::build_recipient_list(undef,
@@ -595,17 +949,40 @@ sub send_mail {
if ($totalcount>2500) {
$emailto.=",$perlvar{'lonSysEMail'}";
}
- my $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices";
-
- my $result=system("metasend -b -S 4000000 -t $emailto -s '$subj' -f $statusdir/index.html -m text/html >& /dev/null");
- if ($result != 0) {
- $result=system("mail -s '$subj' $emailto < $statusdir/index.html");
+ my $from;
+ my $hostname=`/bin/hostname`;
+ chop($hostname);
+ $hostname=~s/[^\w\.]//g;
+ if ($hostname) {
+ $from = 'www@'.$hostname;
+ }
+ my $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices";
+ my $loncronmail = "To: $emailto\n".
+ "From: $from\n".
+ "Subject: ".$subj."\n".
+ "Content-type: text/html\; charset=UTF-8\n".
+ "MIME-Version: 1.0\n\n";
+ if (open(my $fh,"<$statusdir/index.html")) {
+ while (<$fh>) {
+ $loncronmail .= $_;
+ }
+ close($fh);
+ } else {
+ $loncronmail .= "Failed to read from http://$hostname/lon-status/index.html\n";
+ }
+ $loncronmail .= "\n\n";
+ if (open(my $mailh, "|/usr/lib/sendmail -oi -t -odb")) {
+ print $mailh $loncronmail;
+ close($mailh);
+ print "Sending mail.\n";
+ } else {
+ print "Sending mail failed.\n";
}
}
sub usage {
print(< \$help,
"justcheckdaemons" => \$justcheckdaemons,
"noemail" => \$noemail,
"justcheckconnections" => \$justcheckconnections,
- "justreload" => \$justreload
+ "justreload" => \$justreload,
+ "justiptables" => \$justiptables
);
if ($help) { &usage(); return; }
# --------------------------------- Read loncapa_apache.conf and loncapa.conf
@@ -653,26 +1034,42 @@ sub main () {
chop $hostname;
$hostname=~s/[^\w\.]//g; # make sure is safe to pass through shell
my $subj="LON: Unconfigured machine $hostname";
- system("echo 'Unconfigured machine $hostname.' |\
- mailto $emailto -s '$subj' > /dev/null");
+ system("echo 'Unconfigured machine $hostname.' |".
+ " mail -s '$subj' $emailto > /dev/null");
exit 1;
}
# ----------------------------- Make sure this process is running from user=www
my $wwwid=getpwnam('www');
if ($wwwid!=$<) {
- print("User ID mismatch. This program must be run as user 'www'\n");
+ print("User ID mismatch. This program must be run as user 'www'.\n");
my $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}";
my $subj="LON: $perlvar{'lonHostID'} User ID mismatch";
- system("echo 'User ID mismatch. loncron must be run as user www.' |\
- mailto $emailto -s '$subj' > /dev/null");
+ system("echo 'User ID mismatch. loncron must be run as user www.' |".
+ " mail -s '$subj' $emailto > /dev/null");
exit 1;
}
# -------------------------------------------- Force reload of host information
- &Apache::lonnet::load_hosts_tab(1);
- &Apache::lonnet::load_domain_tab(1);
- &Apache::lonnet::get_iphost(1);
+ my $nomemcache;
+ if ($justcheckdaemons) {
+ $nomemcache=1;
+ my $memcachepidfile="$perlvar{'lonDaemons'}/logs/memcached.pid";
+ my $memcachepid;
+ if (-e $memcachepidfile) {
+ my $memfh=IO::File->new($memcachepidfile);
+ $memcachepid=<$memfh>;
+ chomp($memcachepid);
+ if ($memcachepid =~ /^\d+$/ && kill 0 => $memcachepid) {
+ undef($nomemcache);
+ }
+ }
+ }
+ if (!$justiptables) {
+ &Apache::lonnet::load_hosts_tab(1,$nomemcache);
+ &Apache::lonnet::load_domain_tab(1,$nomemcache);
+ &Apache::lonnet::get_iphost(1,$nomemcache);
+ }
# ----------------------------------------- Force firewall update for lond port
@@ -686,9 +1083,12 @@ sub main () {
print $fh "$key\n";
}
close($fh);
- my $execpath = $perlvar{'lonDaemons'}.'/lciptables';
- system("$execpath $tmpfile");
- unlink($fh);
+ if (&LONCAPA::try_to_lock('/tmp/lock_lciptables')) {
+ my $execpath = $perlvar{'lonDaemons'}.'/lciptables';
+ system("$execpath $tmpfile");
+ unlink('/tmp/lock_lciptables'); # Remove the lock file.
+ }
+ unlink($tmpfile);
}
}
@@ -700,17 +1100,19 @@ sub main () {
my $fh;
- if (!$justcheckdaemons && !$justcheckconnections && !$justreload) {
+ if (!$justcheckdaemons && !$justcheckconnections && !$justreload && !$justiptables) {
$fh=&start_logging();
&log_machine_info($fh);
&clean_tmp($fh);
&clean_lonIDs($fh);
+ &clean_balanceIDs($fh);
+ &clean_webDAV_sessionIDs($fh);
&check_httpd_logs($fh);
&rotate_lonnet_logs($fh);
&rotate_other_logs($fh);
}
- if (!$justcheckconnections && !$justreload) {
+ if (!$justcheckconnections && !$justreload && !$justiptables) {
&checkon_daemon($fh,'lonmemcached',40000);
&checkon_daemon($fh,'lonsql',200000);
if ( &checkon_daemon($fh,'lond',40000,'USR1') eq 'running') {
@@ -727,11 +1129,14 @@ sub main () {
if ($justcheckconnections) {
&test_connections($fh);
}
- if (!$justcheckdaemons && !$justcheckconnections && !$justreload) {
+ if (!$justcheckdaemons && !$justcheckconnections && !$justreload && !$justiptables) {
&check_delayed_msg($fh);
&finish_logging($fh);
&log_simplestatus();
-
+ &write_loncaparevs();
+ &write_serverhomeIDs();
+ &write_checksums();
+ &write_hostips();
if ($totalcount>200 && !$noemail) { &send_mail(); }
}
}
@@ -739,10 +1144,3 @@ sub main () {
&main();
1;
-
-
-
-
-
-
-