--- loncom/loncron 2018/12/22 20:05:16 1.116
+++ loncom/loncron 2021/02/02 21:14:36 1.128
@@ -2,7 +2,7 @@
# Housekeeping program, started by cron, loncontrol and loncron.pl
#
-# $Id: loncron,v 1.116 2018/12/22 20:05:16 raeburn Exp $
+# $Id: loncron,v 1.128 2021/02/02 21:14:36 raeburn Exp $
#
# Copyright Michigan State University Board of Trustees
#
@@ -42,9 +42,12 @@ use IO::File;
use IO::Socket;
use HTML::Entities;
use Getopt::Long;
-use GDBM_File;
+use GDBM_File qw(GDBM_READER);
use Storable qw(thaw);
use File::ReadBackwards;
+use File::Copy;
+use Sys::Hostname::FQDN();
+
#globals
use vars qw (%perlvar %simplestatus $errors $warnings $notices $totalcount);
@@ -77,14 +80,14 @@ sub rotate_logfile {
rename("$file.2","$file.3");
rename("$file.1","$file.2");
rename("$file","$file.1");
- }
+ }
}
sub start_daemon {
my ($fh,$daemon,$pidfile,$args) = @_;
my $progname=$daemon;
if ($daemon eq 'lonc') {
- $progname='loncnew';
+ $progname='loncnew';
}
my $error_fname="$perlvar{'lonDaemons'}/logs/${daemon}_errors";
&rotate_logfile($error_fname,$fh,'error logs');
@@ -118,7 +121,7 @@ sub checkon_daemon {
if ($fh) {
if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){
if (open(DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/$daemon.log|")) {
- while (my $line=<DFH>) {
+ while (my $line=<DFH>) {
&log($fh,"$line");
if ($line=~/INFO/) { $notices++; }
if ($line=~/WARNING/) { $notices++; }
@@ -129,9 +132,9 @@ sub checkon_daemon {
}
&log($fh,"</tt></p>");
}
-
+
my $pidfile="$perlvar{'lonDaemons'}/logs/$daemon.pid";
-
+
my $restartflag=1;
my $daemonpid;
if (-e $pidfile) {
@@ -214,7 +217,7 @@ sub checkon_daemon {
}
}
}
-
+
my $fname="$perlvar{'lonDaemons'}/logs/$daemon.log";
&rotate_logfile($fname,$fh,'logs');
@@ -227,19 +230,34 @@ sub log_machine_info {
my ($fh)=@_;
&log($fh,'<hr /><a name="machine" /><h2>Machine Information</h2>');
&log($fh,"<h3>loadavg</h3>");
-
+
+ my $cpucount;
+ if (open(PIPE,"lscpu |grep '^CPU(s)' 2>&1 |")) {
+ my $info = <PIPE>;
+ chomp($info);
+ ($cpucount) = ($info =~ /^\QCPU(s):\E\s+(\d+)$/);
+ close(PIPE);
+ }
+ if (!$cpucount) {
+ $cpucount = 1;
+ }
+ my %loadtarget = (
+ error => 4.0*$cpucount,
+ warn => 2.0*$cpucount,
+ note => 1.0*$cpucount,
+ );
open (LOADAVGH,"/proc/loadavg");
my $loadavg=<LOADAVGH>;
close (LOADAVGH);
-
+
&log($fh,"<tt>$loadavg</tt>");
-
+
my @parts=split(/\s+/,$loadavg);
- if ($parts[1]>4.0) {
+ if ($parts[1]>$loadtarget{'error'}) {
$errors++;
- } elsif ($parts[1]>2.0) {
+ } elsif ($parts[1]>$loadtarget{'warn'}) {
$warnings++;
- } elsif ($parts[1]>1.0) {
+ } elsif ($parts[1]>$loadtarget{'note'}) {
$notices++;
}
@@ -247,14 +265,14 @@ sub log_machine_info {
&log($fh,"<pre>");
open (DFH,"df|");
- while (my $line=<DFH>) {
- &log($fh,&encode_entities($line,'<>&"'));
+ while (my $line=<DFH>) {
+ &log($fh,&encode_entities($line,'<>&"'));
@parts=split(/\s+/,$line);
my $usage=$parts[4];
$usage=~s/\W//g;
- if ($usage>90) {
+ if ($usage>90) {
$warnings++;
- $notices++;
+ $notices++;
} elsif ($usage>80) {
$warnings++;
} elsif ($usage>60) {
@@ -271,8 +289,8 @@ sub log_machine_info {
my $psproc=0;
open (PSH,"ps aux --cols 140 |");
- while (my $line=<PSH>) {
- &log($fh,&encode_entities($line,'<>&"'));
+ while (my $line=<PSH>) {
+ &log($fh,&encode_entities($line,'<>&"'));
$psproc++;
}
close (PSH);
@@ -294,7 +312,7 @@ sub start_logging {
my %simplestatus=();
my $now=time;
my $date=localtime($now);
-
+
&log($fh,(<<ENDHEADERS));
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
@@ -410,12 +428,12 @@ sub recursive_clean_tmp {
($cleaned,$old,$removed) =
&recursive_clean_tmp($innerdir,$cleaned,$old,$removed,$errors);
my @doms = &Apache::lonnet::current_machine_domains();
-
+
if (open(my $dirhandle,$fname)) {
unless (($innerdir eq 'helprequests') ||
(($innerdir =~ /^addcourse/) && ($innerdir !~ m{/\d+$}))) {
my @contents = grep {!/^\.\.?$/} readdir($dirhandle);
- join('&&',@contents)."\n";
+ join('&&',@contents)."\n";
if (scalar(grep {!/^\.\.?$/} readdir($dirhandle)) == 0) {
closedir($dirhandle);
if ($fname =~ m{^\Q$perlvar{'lonDaemons'}\E/tmp/}) {
@@ -468,7 +486,7 @@ sub recursive_clean_tmp {
}
}
} elsif (ref($errors->{failopen}) eq 'ARRAY') {
- push(@{$errors->{failopen}},$fname);
+ push(@{$errors->{failopen}},$fname);
}
} else {
if (unlink($fname)) {
@@ -494,19 +512,38 @@ sub clean_lonIDs {
my $cleaned=0;
my $active=0;
while (my $fname=<$perlvar{'lonIDsDir'}/*>) {
- my ($dev,$ino,$mode,$nlink,
- $uid,$gid,$rdev,$size,
- $atime,$mtime,$ctime,
- $blksize,$blocks)=stat($fname);
- my $now=time;
- my $since=$now-$mtime;
- if ($since>$perlvar{'lonExpire'}) {
- $cleaned++;
- &log($fh,"Unlinking $fname<br />");
- unlink("$fname");
- } else {
- $active++;
- }
+ my $now=time;
+ if (-l $fname) {
+ my $linkfname = readlink($fname);
+ if (-f $linkfname) {
+ if ($linkfname =~ m{^$perlvar{'lonIDsDir'}/[^/]+\.id$}) {
+ my @data = stat($linkfname);
+ my $mtime = $data[9];
+ my $since=$now-$mtime;
+ if ($since>$perlvar{'lonExpire'}) {
+ if (unlink($linkfname)) {
+ $cleaned++;
+ &log($fh,"Unlinking $linkfname<br />");
+ unlink($fname);
+ }
+ }
+ }
+ } else {
+ unlink($fname);
+ }
+ } elsif (-f $fname) {
+ my @data = stat($fname);
+ my $mtime = $data[9];
+ my $since=$now-$mtime;
+ if ($since>$perlvar{'lonExpire'}) {
+ if (unlink($fname)) {
+ $cleaned++;
+ &log($fh,"Unlinking $fname<br />");
+ }
+ } else {
+ $active++;
+ }
+ }
}
&log($fh,"<p>Cleaned up ".$cleaned." stale session token(s).</p>");
&log($fh,"<h3>$active open session(s)</h3>");
@@ -520,7 +557,7 @@ sub clean_balanceIDs {
my $cleaned=0;
my $active=0;
if (-d $perlvar{'lonBalanceDir'}) {
- while (my $fname=<$perlvar{'balanceDir'}/*.id>) {
+ while (my $fname=<$perlvar{'lonBalanceDir'}/*.id>) {
my ($dev,$ino,$mode,$nlink,
$uid,$gid,$rdev,$size,
$atime,$mtime,$ctime,
@@ -566,6 +603,34 @@ sub clean_webDAV_sessionIDs {
}
}
+# ------------------------------------------------------------ clean out ltiIDs
+
+sub clean_ltiIDs {
+ my ($fh)=@_;
+ &log($fh,'<hr /><a name="ltisessions" /><h2>LTI Session Pointers</h2>');
+ my $cleaned=0;
+ my $active=0;
+ if (-d $perlvar{'ltiIDsDir'}) {
+ while (my $fname=<$perlvar{'ltiIDsDir'}/*>) {
+ my ($dev,$ino,$mode,$nlink,
+ $uid,$gid,$rdev,$size,
+ $atime,$mtime,$ctime,
+ $blksize,$blocks)=stat($fname);
+ my $now=time;
+ my $since=$now-$mtime;
+ if ($since>$perlvar{'lonExpire'}) {
+ $cleaned++;
+ &log($fh,"Unlinking $fname<br />");
+ unlink("$fname");
+ } else {
+ $active++;
+ }
+ }
+ }
+ &log($fh,"<p>Cleaned up ".$cleaned." old LTI session pointers.</p>");
+ &log($fh,"<h3>$active unexpired LTI session pointers</h3>");
+}
+
# ----------------------------------------------------------- clean out sockets
sub clean_sockets {
my ($fh)=@_;
@@ -603,16 +668,16 @@ sub rotate_lonnet_logs {
print "Checking logs.\n";
if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
- while (my $line=<DFH>) {
+ while (my $line=<DFH>) {
&log($fh,&encode_entities($line,'<>&"'));
}
close (DFH);
}
&log($fh,"</pre><h3>Perm Log</h3><pre>");
-
+
if (-e "$perlvar{'lonDaemons'}/logs/lonnet.perm.log") {
open(DFH,"tail -n10 $perlvar{'lonDaemons'}/logs/lonnet.perm.log|");
- while (my $line=<DFH>) {
+ while (my $line=<DFH>) {
&log($fh,&encode_entities($line,'<>&"'));
}
close (DFH);
@@ -982,7 +1047,7 @@ sub write_serverhomeIDs {
eval {
local $SIG{ ALRM } = sub { die "TIMEOUT" };
alarm(10);
- $serverhomeID =
+ $serverhomeID =
&Apache::lonnet::get_server_homeID($name,1,'loncron');
alarm(0);
};
@@ -1027,9 +1092,97 @@ sub write_checksums {
return;
}
+sub write_hostips {
+ my $lontabdir = $perlvar{'lonTabDir'};
+ my $defdom = $perlvar{'lonDefDomain'};
+ my $lonhost = $perlvar{'lonHostID'};
+ my $newfile = "$lontabdir/currhostips.tab";
+ my $oldfile = "$lontabdir/prevhostips.tab";
+ my (%prevhosts,%currhosts,%ipchange);
+ if ((-e $newfile) && (-s $newfile)) {
+ move($newfile,$oldfile);
+ chmod(0644,$oldfile);
+ if (open(my $fh,'<',$oldfile)) {
+ while (my $line=<$fh>) {
+ chomp($line);
+ if ($line =~ /^([^:]+):([\d.]+)$/) {
+ $prevhosts{$1} = $2;
+ }
+ }
+ close($fh);
+ }
+ }
+ my ($ip_info,$cached) =
+ &Apache::lonnet::is_cached_new('iphost','iphost');
+ if (!$cached) {
+ &Apache::lonnet::get_iphost();
+ ($ip_info,$cached) =
+ &Apache::lonnet::is_cached_new('iphost','iphost');
+ }
+ if (ref($ip_info) eq 'ARRAY') {
+ %currhosts = %{$ip_info->[1]};
+ if (open(my $fh,'>',$newfile)) {
+ foreach my $key (keys(%currhosts)) {
+ print $fh "$key:$currhosts{$key}\n";
+ }
+ close($fh);
+ chmod(0644,$newfile);
+ }
+ }
+ if (keys(%prevhosts) && keys(%currhosts)) {
+ foreach my $key (keys(%prevhosts)) {
+ unless ($currhosts{$key} eq $prevhosts{$key}) {
+ $ipchange{$key} = $prevhosts{$key}.' | '.$currhosts{$key};
+ }
+ }
+ foreach my $key (keys(%currhosts)) {
+ unless ($currhosts{$key} eq $prevhosts{$key}) {
+ $ipchange{$key} = $prevhosts{$key}.' | '.$currhosts{$key};
+ }
+ }
+ }
+ if (&Apache::lonnet::domain($defdom,'primary') eq $lonhost) {
+ if (keys(%ipchange)) {
+ if (open(my $fh,'>>',$perlvar{'lonDaemons'}.'/logs/hostip.log')) {
+ print $fh "********************\n".localtime(time).' Changes --'."\n".
+ "| Hostname | Previous IP | New IP |\n".
+ " --------------------------------- \n";
+ foreach my $hostname (sort(keys(%ipchange))) {
+ print $fh "| $hostname | $ipchange{$hostname} |\n";
+ }
+ print $fh "\n*******************\n\n";
+ close($fh);
+ }
+ my $emailto = &Apache::loncommon::build_recipient_list(undef,
+ 'hostipmail',$defdom);
+ if ($emailto) {
+ my $subject = "LON-CAPA Hostname to IP change ($perlvar{'lonHostID'})";
+ my $chgmail = "To: $emailto\n".
+ "Subject: $subject\n".
+ "Content-type: text/plain\; charset=UTF-8\n".
+ "MIME-Version: 1.0\n\n".
+ "Host/IP changes\n".
+ " \n".
+ "| Hostname | Previous IP | New IP |\n".
+ " --------------------------------- \n";
+ foreach my $hostname (sort(keys(%ipchange))) {
+ $chgmail .= "| $hostname | $ipchange{$hostname} |\n";
+ }
+ $chgmail .= "\n\n";
+ if (open(my $mailh, "|/usr/lib/sendmail -oi -t -odb")) {
+ print $mailh $chgmail;
+ close($mailh);
+ print "Sending mail notification of hostname/IP changes.\n";
+ }
+ }
+ }
+ }
+ return;
+}
+
sub clean_nosslverify {
my ($fh) = @_;
- my %unlinked;
+ my %unlinked;
if (-d "$perlvar{'lonSockDir'}/nosslverify") {
if (opendir(my $dh,"$perlvar{'lonSockDir'}/nosslverify")) {
while (my $fname=readdir($dh)) {
@@ -1191,7 +1344,7 @@ sub write_hosttypes {
foreach my $lonid (sort(keys(%hostdom))) {
my $type = 'other';
if ($hostdom{$lonid} eq $dom) {
- $type = 'dom';
+ $type = 'dom';
} elsif ($intdom{$lonid} eq $internetdom) {
$type = 'intdom';
}
@@ -1397,19 +1550,23 @@ Options:
do not send emails do not
check if the daemons are running, do not
generate lon-status
-
+ --justiptables Only update the dynamic iptables rules for the
+ lond port; do not send emails, do not
+ check if the daemons are running, do not
+ generate lon-status
USAGE
}
# ================================================================ Main Program
sub main () {
my ($help,$justcheckdaemons,$noemail,$justcheckconnections,
- $justreload);
+ $justreload,$justiptables);
&GetOptions("help" => \$help,
"justcheckdaemons" => \$justcheckdaemons,
"noemail" => \$noemail,
"justcheckconnections" => \$justcheckconnections,
- "justreload" => \$justreload
+ "justreload" => \$justreload,
+ "justiptables" => \$justiptables
);
if ($help) { &usage(); return; }
# --------------------------------- Read loncapa_apache.conf and loncapa.conf
@@ -1424,9 +1581,10 @@ sub main () {
if ('{[[[[lonHostID]]]]}' eq $perlvar{'lonHostID'}) {
print("Unconfigured machine.\n");
my $emailto=$perlvar{'lonSysEMail'};
- my $hostname=`/bin/hostname`;
- chop $hostname;
- $hostname=~s/[^\w\.]//g; # make sure is safe to pass through shell
+ my $hostname = Sys::Hostname::FQDN::fqdn();
+ $hostname=~s/\.+/./g;
+ $hostname=~s/\-+/-/g;
+ $hostname=~s/[^\w\.-]//g; # make sure is safe to pass through shell
my $subj="LON: Unconfigured machine $hostname";
system("echo 'Unconfigured machine $hostname.' |".
" mail -s '$subj' $emailto > /dev/null");
@@ -1459,11 +1617,13 @@ sub main () {
}
}
}
- &Apache::lonnet::load_hosts_tab(1,$nomemcache);
- &Apache::lonnet::load_domain_tab(1,$nomemcache);
- &Apache::lonnet::get_iphost(1,$nomemcache);
+ if (!$justiptables) {
+ &Apache::lonnet::load_hosts_tab(1,$nomemcache);
+ &Apache::lonnet::load_domain_tab(1,$nomemcache);
+ &Apache::lonnet::get_iphost(1,$nomemcache);
+ }
-# ----------------------------------------- Force firewall update for lond port
+# ----------------------------------------- Force firewall update for lond port
if ((!$justcheckdaemons) && (!$justreload)) {
my $now = time;
@@ -1478,7 +1638,7 @@ sub main () {
if (&LONCAPA::try_to_lock('/tmp/lock_lciptables')) {
my $execpath = $perlvar{'lonDaemons'}.'/lciptables';
system("$execpath $tmpfile");
- unlink('/tmp/lock_lciptables'); # Remove the lock file.
+ unlink('/tmp/lock_lciptables'); # Remove the lock file.
}
unlink($tmpfile);
}
@@ -1490,9 +1650,9 @@ sub main () {
$warnings=0;
$notices=0;
-
+
my $fh;
- if (!$justcheckdaemons && !$justcheckconnections && !$justreload) {
+ if (!$justcheckdaemons && !$justcheckconnections && !$justreload && !$justiptables) {
$fh=&start_logging();
&log_machine_info($fh);
@@ -1500,11 +1660,12 @@ sub main () {
&clean_lonIDs($fh);
&clean_balanceIDs($fh);
&clean_webDAV_sessionIDs($fh);
+ &clean_ltiIDs($fh);
&check_httpd_logs($fh);
&rotate_lonnet_logs($fh);
&rotate_other_logs($fh);
}
- if (!$justcheckconnections && !$justreload) {
+ if (!$justcheckconnections && !$justreload && !$justiptables) {
&checkon_daemon($fh,'lonmemcached',40000);
&checkon_daemon($fh,'lonsql',200000);
if ( &checkon_daemon($fh,'lond',40000,'USR1') eq 'running') {
@@ -1525,7 +1686,7 @@ sub main () {
if ($justcheckconnections) {
&test_connections($fh);
}
- if (!$justcheckdaemons && !$justcheckconnections && !$justreload) {
+ if (!$justcheckdaemons && !$justcheckconnections && !$justreload && !$justiptables) {
my $domconf = &get_domain_config();
my ($threshold,$sysmail,$reportstatus,$weightsref,$exclusionsref) =
&get_permcount_settings($domconf);
@@ -1533,6 +1694,7 @@ sub main () {
&write_loncaparevs();
&write_serverhomeIDs();
&write_checksums();
+ &write_hostips();
my %sslrem = &clean_nosslverify($fh);
my %conchgs = &write_connection_config();
my %hosttypechgs = &write_hosttypes();