--- loncom/loncron 2019/03/17 23:23:21 1.117 +++ loncom/loncron 2021/02/02 21:14:36 1.128 @@ -2,7 +2,7 @@ # Housekeeping program, started by cron, loncontrol and loncron.pl # -# $Id: loncron,v 1.117 2019/03/17 23:23:21 raeburn Exp $ +# $Id: loncron,v 1.128 2021/02/02 21:14:36 raeburn Exp $ # # Copyright Michigan State University Board of Trustees # @@ -42,10 +42,12 @@ use IO::File; use IO::Socket; use HTML::Entities; use Getopt::Long; -use GDBM_File; +use GDBM_File qw(GDBM_READER); use Storable qw(thaw); use File::ReadBackwards; use File::Copy; +use Sys::Hostname::FQDN(); + #globals use vars qw (%perlvar %simplestatus $errors $warnings $notices $totalcount); @@ -78,14 +80,14 @@ sub rotate_logfile { rename("$file.2","$file.3"); rename("$file.1","$file.2"); rename("$file","$file.1"); - } + } } sub start_daemon { my ($fh,$daemon,$pidfile,$args) = @_; my $progname=$daemon; if ($daemon eq 'lonc') { - $progname='loncnew'; + $progname='loncnew'; } my $error_fname="$perlvar{'lonDaemons'}/logs/${daemon}_errors"; &rotate_logfile($error_fname,$fh,'error logs'); @@ -119,7 +121,7 @@ sub checkon_daemon { if ($fh) { if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ if (open(DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/$daemon.log|")) { - while (my $line=) { + while (my $line=) { &log($fh,"$line"); if ($line=~/INFO/) { $notices++; } if ($line=~/WARNING/) { $notices++; } @@ -130,9 +132,9 @@ sub checkon_daemon { } &log($fh,"

"); } - + my $pidfile="$perlvar{'lonDaemons'}/logs/$daemon.pid"; - + my $restartflag=1; my $daemonpid; if (-e $pidfile) { @@ -215,7 +217,7 @@ sub checkon_daemon { } } } - + my $fname="$perlvar{'lonDaemons'}/logs/$daemon.log"; &rotate_logfile($fname,$fh,'logs'); @@ -228,19 +230,34 @@ sub log_machine_info { my ($fh)=@_; &log($fh,'

Machine Information

'); &log($fh,"

loadavg

"); - + + my $cpucount; + if (open(PIPE,"lscpu |grep '^CPU(s)' 2>&1 |")) { + my $info = ; + chomp($info); + ($cpucount) = ($info =~ /^\QCPU(s):\E\s+(\d+)$/); + close(PIPE); + } + if (!$cpucount) { + $cpucount = 1; + } + my %loadtarget = ( + error => 4.0*$cpucount, + warn => 2.0*$cpucount, + note => 1.0*$cpucount, + ); open (LOADAVGH,"/proc/loadavg"); my $loadavg=; close (LOADAVGH); - + &log($fh,"$loadavg"); - + my @parts=split(/\s+/,$loadavg); - if ($parts[1]>4.0) { + if ($parts[1]>$loadtarget{'error'}) { $errors++; - } elsif ($parts[1]>2.0) { + } elsif ($parts[1]>$loadtarget{'warn'}) { $warnings++; - } elsif ($parts[1]>1.0) { + } elsif ($parts[1]>$loadtarget{'note'}) { $notices++; } @@ -248,14 +265,14 @@ sub log_machine_info { &log($fh,"
");
 
     open (DFH,"df|");
-    while (my $line=) { 
-	&log($fh,&encode_entities($line,'<>&"')); 
+    while (my $line=) {
+	&log($fh,&encode_entities($line,'<>&"'));
 	@parts=split(/\s+/,$line);
 	my $usage=$parts[4];
 	$usage=~s/\W//g;
-	if ($usage>90) { 
+	if ($usage>90) {
 	    $warnings++;
-	    $notices++; 
+	    $notices++;
 	} elsif ($usage>80) {
 	    $warnings++;
 	} elsif ($usage>60) {
@@ -272,8 +289,8 @@ sub log_machine_info {
     my $psproc=0;
 
     open (PSH,"ps aux --cols 140 |");
-    while (my $line=) { 
-	&log($fh,&encode_entities($line,'<>&"')); 
+    while (my $line=) {
+	&log($fh,&encode_entities($line,'<>&"'));
 	$psproc++;
     }
     close (PSH);
@@ -295,7 +312,7 @@ sub start_logging {
     my %simplestatus=();
     my $now=time;
     my $date=localtime($now);
-    
+ 
 
     &log($fh,(<
@@ -411,12 +428,12 @@ sub recursive_clean_tmp {
                 ($cleaned,$old,$removed) = 
                      &recursive_clean_tmp($innerdir,$cleaned,$old,$removed,$errors);
                 my @doms = &Apache::lonnet::current_machine_domains();
-                
+
                 if (open(my $dirhandle,$fname)) {
                     unless (($innerdir eq 'helprequests') ||
                             (($innerdir =~ /^addcourse/) && ($innerdir !~ m{/\d+$}))) {
                         my @contents = grep {!/^\.\.?$/} readdir($dirhandle);
-                                      join('&&',@contents)."\n";    
+                                      join('&&',@contents)."\n";
                         if (scalar(grep {!/^\.\.?$/} readdir($dirhandle)) == 0) {
                             closedir($dirhandle);
                             if ($fname =~ m{^\Q$perlvar{'lonDaemons'}\E/tmp/}) {
@@ -469,7 +486,7 @@ sub recursive_clean_tmp {
                                 }
                             }
                         } elsif (ref($errors->{failopen}) eq 'ARRAY') {
-                            push(@{$errors->{failopen}},$fname); 
+                            push(@{$errors->{failopen}},$fname);
                         }
                     } else {
                         if (unlink($fname)) {
@@ -495,19 +512,38 @@ sub clean_lonIDs {
     my $cleaned=0;
     my $active=0;
     while (my $fname=<$perlvar{'lonIDsDir'}/*>) {
-	my ($dev,$ino,$mode,$nlink,
-	    $uid,$gid,$rdev,$size,
-	    $atime,$mtime,$ctime,
-	    $blksize,$blocks)=stat($fname);
-	my $now=time;
-	my $since=$now-$mtime;
-	if ($since>$perlvar{'lonExpire'}) {
-	    $cleaned++;
-	    &log($fh,"Unlinking $fname
");
-	    unlink("$fname");
-	} else {
-	    $active++;
-	}
+        my $now=time;
+        if (-l $fname) {
+            my $linkfname = readlink($fname);
+            if (-f $linkfname) {
+                if ($linkfname =~ m{^$perlvar{'lonIDsDir'}/[^/]+\.id$}) {
+                    my @data = stat($linkfname);
+                    my $mtime = $data[9];
+                    my $since=$now-$mtime;
+                    if ($since>$perlvar{'lonExpire'}) {
+                        if (unlink($linkfname)) {
+                            $cleaned++;
+                            &log($fh,"Unlinking $linkfname
");
+                            unlink($fname);
+                        }
+                    }
+                }
+            } else {
+               unlink($fname);
+            }
+        } elsif (-f $fname) {
+            my @data = stat($fname);
+            my $mtime = $data[9];
+            my $since=$now-$mtime;
+            if ($since>$perlvar{'lonExpire'}) {
+                if (unlink($fname)) {
+                    $cleaned++;
+                    &log($fh,"Unlinking $fname
");
+                }
+            } else {
+                $active++;
+            }
+        }
     }
     &log($fh,"
Cleaned up ".$cleaned." stale session token(s).
");
     &log($fh,"
$active open session(s)
");
@@ -521,7 +557,7 @@ sub clean_balanceIDs {
     my $cleaned=0;
     my $active=0;
     if (-d $perlvar{'lonBalanceDir'}) {
-        while (my $fname=<$perlvar{'balanceDir'}/*.id>) {
+        while (my $fname=<$perlvar{'lonBalanceDir'}/*.id>) {
             my ($dev,$ino,$mode,$nlink,
                 $uid,$gid,$rdev,$size,
                 $atime,$mtime,$ctime,
@@ -567,6 +603,34 @@ sub clean_webDAV_sessionIDs {
     }
 }
 
+# ------------------------------------------------------------ clean out ltiIDs
+
+sub clean_ltiIDs {
+    my ($fh)=@_;
+    &log($fh,'
LTI Session Pointers
');
+    my $cleaned=0;
+    my $active=0;
+    if (-d $perlvar{'ltiIDsDir'}) {
+        while (my $fname=<$perlvar{'ltiIDsDir'}/*>) {
+            my ($dev,$ino,$mode,$nlink,
+                $uid,$gid,$rdev,$size,
+                $atime,$mtime,$ctime,
+                $blksize,$blocks)=stat($fname);
+            my $now=time;
+            my $since=$now-$mtime;
+            if ($since>$perlvar{'lonExpire'}) {
+                $cleaned++;
+                &log($fh,"Unlinking $fname
");
+                unlink("$fname");
+            } else {
+                $active++;
+            }
+        }
+    }
+    &log($fh,"
Cleaned up ".$cleaned." old LTI session pointers.
");
+    &log($fh,"
$active unexpired LTI session pointers
");
+}
+
 # ----------------------------------------------------------- clean out sockets
 sub clean_sockets {
     my ($fh)=@_;
@@ -604,16 +668,16 @@ sub rotate_lonnet_logs {
     print "Checking logs.\n";
     if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
 	open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
-	while (my $line=) { 
+	while (my $line=) {
 	    &log($fh,&encode_entities($line,'<>&"'));
 	}
 	close (DFH);
     }
     &log($fh,"

Perm Log

");
-    
+
     if (-e "$perlvar{'lonDaemons'}/logs/lonnet.perm.log") {
 	open(DFH,"tail -n10 $perlvar{'lonDaemons'}/logs/lonnet.perm.log|");
-	while (my $line=) { 
+	while (my $line=) {
 	    &log($fh,&encode_entities($line,'<>&"'));
 	}
 	close (DFH);
@@ -983,7 +1047,7 @@ sub write_serverhomeIDs {
                 eval {
                     local $SIG{ ALRM } = sub { die "TIMEOUT" };
                     alarm(10);
-                    $serverhomeID = 
+                    $serverhomeID =
                         &Apache::lonnet::get_server_homeID($name,1,'loncron');
                     alarm(0);
                 };
@@ -1068,7 +1132,7 @@ sub write_hostips {
     if (keys(%prevhosts) && keys(%currhosts)) {
         foreach my $key (keys(%prevhosts)) {
             unless ($currhosts{$key} eq $prevhosts{$key}) {
-                $ipchange{$key} = $prevhosts{$key}.'|'.$currhosts{$key};
+                $ipchange{$key} = $prevhosts{$key}.' | '.$currhosts{$key};
             }
         }
         foreach my $key (keys(%currhosts)) {
@@ -1081,10 +1145,10 @@ sub write_hostips {
         if (keys(%ipchange)) {
             if (open(my $fh,'>>',$perlvar{'lonDaemons'}.'/logs/hostip.log')) {
                print $fh "********************\n".localtime(time).' Changes --'."\n".
-                         "Hostname | Previous IP | New IP\n".
-                         "--------------------------------\n";
+                         "| Hostname | Previous IP | New IP |\n".
+                         " --------------------------------- \n";
                foreach my $hostname (sort(keys(%ipchange))) {
-                    print $fh "$hostname | $ipchange{$hostname}\n";
+                    print $fh "| $hostname | $ipchange{$hostname} |\n";
                 }
                 print $fh "\n*******************\n\n";
                 close($fh);
@@ -1099,10 +1163,10 @@ sub write_hostips {
                               "MIME-Version: 1.0\n\n".
                               "Host/IP changes\n".
                               " \n".
-                              "Hostname | Previous IP | New IP\n".
-                              "--------------------------------\n";
+                              "| Hostname | Previous IP | New IP |\n".
+                              " --------------------------------- \n";
                 foreach my $hostname (sort(keys(%ipchange))) {
-                    $chgmail .= "$hostname | $ipchange{$hostname}\n";
+                    $chgmail .= "| $hostname | $ipchange{$hostname} |\n";
                 }
                 $chgmail .= "\n\n";
                 if (open(my $mailh, "|/usr/lib/sendmail -oi -t -odb")) {
@@ -1118,7 +1182,7 @@ sub write_hostips {
 
 sub clean_nosslverify {
     my ($fh) = @_;
-    my %unlinked; 
+    my %unlinked;
     if (-d "$perlvar{'lonSockDir'}/nosslverify") {
         if (opendir(my $dh,"$perlvar{'lonSockDir'}/nosslverify")) {
             while (my $fname=readdir($dh)) {
@@ -1280,7 +1344,7 @@ sub write_hosttypes {
                 foreach my $lonid (sort(keys(%hostdom))) {
                     my $type = 'other';
                     if ($hostdom{$lonid} eq $dom) {
-                        $type = 'dom'; 
+                        $type = 'dom';
                     } elsif ($intdom{$lonid} eq $internetdom) {
                         $type = 'intdom';
                     }
@@ -1486,19 +1550,23 @@ Options:
 				do not send emails do not
                                 check if the daemons are running, do not
                                 generate lon-status
-                           
+   --justiptables          Only update the dynamic iptables rules for the
+                                lond port; do not send emails, do not
+                                check if the daemons are running, do not
+                                generate lon-status
 USAGE
 }
 
 # ================================================================ Main Program
 sub main () {
     my ($help,$justcheckdaemons,$noemail,$justcheckconnections,
-	$justreload);
+	$justreload,$justiptables);
     &GetOptions("help"                 => \$help,
 		"justcheckdaemons"     => \$justcheckdaemons,
 		"noemail"              => \$noemail,
 		"justcheckconnections" => \$justcheckconnections,
-		"justreload"           => \$justreload
+		"justreload"           => \$justreload,
+                "justiptables"         => \$justiptables
 		);
     if ($help) { &usage(); return; }
 # --------------------------------- Read loncapa_apache.conf and loncapa.conf
@@ -1513,9 +1581,10 @@ sub main () {
     if ('{[[[[lonHostID]]]]}' eq $perlvar{'lonHostID'}) {
 	print("Unconfigured machine.\n");
 	my $emailto=$perlvar{'lonSysEMail'};
-	my $hostname=`/bin/hostname`;
-	chop $hostname;
-	$hostname=~s/[^\w\.]//g; # make sure is safe to pass through shell
+	my $hostname = Sys::Hostname::FQDN::fqdn();
+	$hostname=~s/\.+/./g;
+	$hostname=~s/\-+/-/g;
+	$hostname=~s/[^\w\.-]//g; # make sure is safe to pass through shell
 	my $subj="LON: Unconfigured machine $hostname";
 	system("echo 'Unconfigured machine $hostname.' |".
                " mail -s '$subj' $emailto > /dev/null");
@@ -1548,11 +1617,13 @@ sub main () {
             }
         }
     }
-    &Apache::lonnet::load_hosts_tab(1,$nomemcache);
-    &Apache::lonnet::load_domain_tab(1,$nomemcache);
-    &Apache::lonnet::get_iphost(1,$nomemcache);
+    if (!$justiptables) {
+        &Apache::lonnet::load_hosts_tab(1,$nomemcache);
+        &Apache::lonnet::load_domain_tab(1,$nomemcache);
+        &Apache::lonnet::get_iphost(1,$nomemcache);
+    }
 
-# ----------------------------------------- Force firewall update for lond port  
+# ----------------------------------------- Force firewall update for lond port
 
     if ((!$justcheckdaemons) && (!$justreload)) {
         my $now = time;
@@ -1567,7 +1638,7 @@ sub main () {
             if (&LONCAPA::try_to_lock('/tmp/lock_lciptables')) {
                 my $execpath = $perlvar{'lonDaemons'}.'/lciptables';
                 system("$execpath $tmpfile");
-                unlink('/tmp/lock_lciptables');  # Remove the lock file. 
+                unlink('/tmp/lock_lciptables');  # Remove the lock file.
             }
             unlink($tmpfile);
         }
@@ -1579,9 +1650,9 @@ sub main () {
     $warnings=0;
     $notices=0;
 
-	
+
     my $fh;
-    if (!$justcheckdaemons && !$justcheckconnections && !$justreload) {
+    if (!$justcheckdaemons && !$justcheckconnections && !$justreload && !$justiptables) {
 	$fh=&start_logging();
 
 	&log_machine_info($fh);
@@ -1589,11 +1660,12 @@ sub main () {
 	&clean_lonIDs($fh);
         &clean_balanceIDs($fh);
         &clean_webDAV_sessionIDs($fh);
+        &clean_ltiIDs($fh);
 	&check_httpd_logs($fh);
 	&rotate_lonnet_logs($fh);
 	&rotate_other_logs($fh);
     }
-    if (!$justcheckconnections && !$justreload) {
+    if (!$justcheckconnections && !$justreload && !$justiptables) {
 	&checkon_daemon($fh,'lonmemcached',40000);
 	&checkon_daemon($fh,'lonsql',200000);
 	if ( &checkon_daemon($fh,'lond',40000,'USR1') eq 'running') {
@@ -1614,7 +1686,7 @@ sub main () {
     if ($justcheckconnections) {
 	&test_connections($fh);
     }
-    if (!$justcheckdaemons && !$justcheckconnections && !$justreload) {
+    if (!$justcheckdaemons && !$justcheckconnections && !$justreload && !$justiptables) {
         my $domconf = &get_domain_config();
         my ($threshold,$sysmail,$reportstatus,$weightsref,$exclusionsref) =
             &get_permcount_settings($domconf);