--- loncom/loncron 2014/03/17 14:47:38 1.100 +++ loncom/loncron 2018/10/29 02:46:41 1.111 @@ -2,7 +2,7 @@ # Housekeeping program, started by cron, loncontrol and loncron.pl # -# $Id: loncron,v 1.100 2014/03/17 14:47:38 bisitz Exp $ +# $Id: loncron,v 1.111 2018/10/29 02:46:41 raeburn Exp $ # # Copyright Michigan State University Board of Trustees # @@ -41,6 +41,8 @@ use IO::File; use IO::Socket; use HTML::Entities; use Getopt::Long; +use GDBM_File; +use Storable qw(thaw); #globals use vars qw (%perlvar %simplestatus $errors $warnings $notices $totalcount); @@ -111,17 +113,20 @@ sub checkon_daemon { my $result; &log($fh,'

'.$daemon.'

Log

'); printf("%-15s ",$daemon); - if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ - open (DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/$daemon.log|"); - while (my $line=) { - &log($fh,"$line"); - if ($line=~/INFO/) { $notices++; } - if ($line=~/WARNING/) { $notices++; } - if ($line=~/CRITICAL/) { $warnings++; } - }; - close (DFH); + if ($fh) { + if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ + if (open(DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/$daemon.log|")) { + while (my $line=) { + &log($fh,"$line"); + if ($line=~/INFO/) { $notices++; } + if ($line=~/WARNING/) { $notices++; } + if ($line=~/CRITICAL/) { $warnings++; } + } + close (DFH); + } + } + &log($fh,"

"); } - &log($fh,"

"); my $pidfile="$perlvar{'lonDaemons'}/logs/$daemon.pid"; @@ -157,14 +162,17 @@ sub checkon_daemon { $errors++; my $kadaemon=$daemon; if ($kadaemon eq 'lonmemcached') { $kadaemon='memcached'; } - &log($fh,'
Killall '.$daemon.': '. + &log($fh,'
Killall '.$daemon.': '. `killall $kadaemon 2>&1`.' - '); sleep 1; &log($fh,unlink($pidfile).' - '. `killall -9 $kadaemon 2>&1`. - '
'); + '
'); + if ($kadaemon eq 'loncnew') { + &clean_lonc_childpids(); + } &log($fh,"

$daemon not running, trying to start

"); - + if (&start_daemon($fh,$daemon,$pidfile,$args)) { &log($fh,"

$daemon at pid $daemonpid responding

"); $simplestatus{$daemon}='restarted'; @@ -189,17 +197,19 @@ sub checkon_daemon { &log($fh,"

Unable to start $daemon

"); } } - - if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ - &log($fh,"

");
-	    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/$daemon.log|");
-	    while (my $line=) { 
-		&log($fh,"$line");
-		if ($line=~/WARNING/) { $notices++; }
-		if ($line=~/CRITICAL/) { $notices++; }
-	    };
-	    close (DFH);
-	    &log($fh,"

"); + if ($fh) { + if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ + &log($fh,"

");
+	        if (open(DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/$daemon.log|")) {
+	            while (my $line=) { 
+		        &log($fh,"$line");
+		        if ($line=~/WARNING/) { $notices++; }
+		        if ($line=~/CRITICAL/) { $notices++; }
+	            }
+	            close (DFH);
+                }
+	        &log($fh,"

"); + } } } @@ -285,9 +295,11 @@ sub start_logging { &log($fh,(< + + LON Status Report $perlvar{'lonHostID'} + @@ -298,6 +310,7 @@ sub start_logging {
  • Machine Information
  • Temporary Files
  • Session Tokens
    • +
  • WebDAV Session Tokens
  • httpd
  • lonsql
  • lond
    • @@ -328,15 +341,20 @@ ENDHEADERS "".$role. "".&Apache::lonnet::hostname($id)."\n"); } - &log($fh,"

    Spare Hosts

    \n"); return $fh; } @@ -482,7 +500,7 @@ sub clean_lonIDs { my $since=$now-$mtime; if ($since>$perlvar{'lonExpire'}) { $cleaned++; - &log($fh,"Unlinking $fname
    "); + &log($fh,"Unlinking $fname
    "); unlink("$fname"); } else { $active++; @@ -492,6 +510,32 @@ sub clean_lonIDs { &log($fh,"

    $active open session(s)

    "); } +# ------------------------------------------------ clean out webDAV Session IDs +sub clean_webDAV_sessionIDs { + my ($fh)=@_; + if ($perlvar{'lonRole'} eq 'library') { + &log($fh,'

    WebDAV Session Tokens

    '); + my $cleaned=0; + my $active=0; + my $now = time; + if (-d $perlvar{'lonDAVsessDir'}) { + while (my $fname=<$perlvar{'lonDAVsessDir'}/*>) { + my @stats = stat($fname); + my $since=$now-$stats[9]; + if ($since>$perlvar{'lonExpire'}) { + $cleaned++; + &log($fh,"Unlinking $fname
    "); + unlink("$fname"); + } else { + $active++; + } + } + &log($fh,"

    Cleaned up ".$cleaned." stale webDAV session token(s).

    "); + &log($fh,"

    $active open webDAV session(s)

    "); + } + } +} + # ----------------------------------------------------------- clean out sockets sub clean_sockets { my ($fh)=@_; @@ -610,12 +654,18 @@ sub check_delayed_msg { my $unsend=0; + my %hostname = &Apache::lonnet::all_hostnames(); + my $numhosts = scalar(keys(%hostname)); + my $dfh=IO::File->new("$perlvar{'lonDaemons'}/logs/lonnet.perm.log"); while (my $line=<$dfh>) { my ($time,$sdf,$dserv,$dcmd)=split(/:/,$line); + if ($numhosts) { + next unless ($hostname{$dserv}); + } if ($sdf eq 'F') { my $local=localtime($time); - &log($fh,"Failed: $time, $dserv, $dcmd
    "); + &log($fh,"Failed: $time, $dserv, $dcmd
    "); $warnings++; } if ($sdf eq 'S') { $unsend--; } @@ -639,8 +689,6 @@ sub check_delayed_msg { } &log($fh,"\n"); close (DFH); - my %hostname = &Apache::lonnet::all_hostnames(); - my $numhosts = scalar(keys(%hostname)); # pong to all servers that have delayed messages # this will trigger a reverse connection, which should flush the buffers foreach my $tryserver (sort(keys(%servers))) { @@ -781,8 +829,265 @@ sub write_checksums { return; } +sub clean_nosslverify { + my ($fh) = @_; + my %unlinked; + if (-d "$perlvar{'lonSockDir'}/nosslverify") { + if (opendir(my $dh,"$perlvar{'lonSockDir'}/nosslverify")) { + while (my $fname=readdir($dh)) { + next if ($fname =~ /^\.+$/); + if (unlink("/home/httpd/sockets/nosslverify/$fname")) { + &log($fh,"Unlinking $fname
    "); + $unlinked{$fname} = 1; + } + } + closedir($dh); + } + } + &log($fh,"

    Removed ".scalar(keys(%unlinked))." nosslverify clients

    "); + return %unlinked; +} +sub clean_lonc_childpids { + my $childpiddir = "$perlvar{'lonDocRoot'}/lon-status/loncchld"; + if (-d $childpiddir) { + if (opendir(my $dh,$childpiddir)) { + while (my $fname=readdir($dh)) { + next if ($fname =~ /^\.+$/); + unlink("$childpiddir/$fname"); + } + closedir($dh); + } + } +} + +sub write_connection_config { + my ($isprimary,$domconf,$url,%connectssl,%changes); + my $primaryLibServer = &Apache::lonnet::domain($perlvar{'lonDefDomain'},'primary'); + if ($primaryLibServer eq $perlvar{'lonHostID'}) { + $isprimary = 1; + } elsif ($primaryLibServer ne '') { + my $protocol = $Apache::lonnet::protocol{$primaryLibServer}; + my $hostname = &Apache::lonnet::hostname($primaryLibServer); + unless ($protocol eq 'https') { + $protocol = 'http'; + } + $url = $protocol.'://'.$hostname.'/cgi-bin/listdomconfig.pl'; + } + my $domconf = &get_domain_config($perlvar{'lonDefDomain'},$primaryLibServer,$isprimary, + $url); + if (ref($domconf) eq 'HASH') { + if (ref($domconf->{'ssl'}) eq 'HASH') { + foreach my $connect ('connto','connfrom') { + if (ref($domconf->{'ssl'}->{$connect}) eq 'HASH') { + my ($sslreq,$sslnoreq,$currsetting); + my %contypes; + foreach my $type ('dom','intdom','other') { + $connectssl{$connect.'_'.$type} = $domconf->{'ssl'}->{$connect}->{$type}; + } + } + } + } + if (keys(%connectssl)) { + my %currconf; + if (open(my $fh,'<',"$perlvar{'lonTabDir'}/connectionrules.tab")) { + while (my $line = <$fh>) { + chomp($line); + my ($name,$value) = split(/=/,$line); + if ($value =~ /^(?:no|yes|req)$/) { + if ($name =~ /^conn(to|from)_(dom|intdom|other)$/) { + $currconf{$name} = $value; + } + } + } + close($fh); + } + if (open(my $fh,'>',"$perlvar{'lonTabDir'}/connectionrules.tab")) { + my $count = 0; + foreach my $key (sort(keys(%connectssl))) { + print $fh "$key=$connectssl{$key}\n"; + if (exists($currconf{$key})) { + unless ($currconf{$key} eq $connectssl{$key}) { + $changes{$key} = 1; + } + } else { + $changes{$key} = 1; + } + $count ++; + } + close($fh); + print "Completed writing SSL options for lonc/lond for $count items.\n"; + } + } else { + print "Writing of SSL options skipped - no connection rules in domain configuration.\n"; + } + } else { + print "Retrieval of SSL options for lonc/lond skipped - no configuration data available for domain.\n"; + } + return %changes; +} + +sub get_domain_config { + my ($dom,$primlibserv,$isprimary,$url) = @_; + my %confhash; + if ($isprimary) { + my $lonusersdir = $perlvar{'lonUsersDir'}; + my $fname = $lonusersdir.'/'.$dom.'/configuration.db'; + if (-e $fname) { + my $dbref=&LONCAPA::locking_hash_tie($fname,&GDBM_READER()); + if (ref($dbref) eq 'HASH') { + foreach my $key (sort(keys(%{$dbref}))) { + my $value = $dbref->{$key}; + if ($value =~ s/^__FROZEN__//) { + $value = thaw(&LONCAPA::unescape($value)); + } else { + $value = &LONCAPA::unescape($value); + } + $confhash{$key} = $value; + } + &LONCAPA::locking_hash_untie($dbref); + } + } + } else { + if (open(PIPE,"wget --no-check-certificate '$url?primary=$primlibserv&format=raw' |")) { + my $config = ''; + while () { + $config .= $_; + } + close(PIPE); + if ($config) { + my @pairs=split(/\&/,$config); + foreach my $item (@pairs) { + my ($key,$value)=split(/=/,$item,2); + my $what = &LONCAPA::unescape($key); + if ($value =~ s/^__FROZEN__//) { + $value = thaw(&LONCAPA::unescape($value)); + } else { + $value = &LONCAPA::unescape($value); + } + $confhash{$what}=$value; + } + } + } + } + return \%confhash; +} + +sub write_hosttypes { + my %intdom = &Apache::lonnet::all_host_intdom(); + my %hostdom = &Apache::lonnet::all_host_domain(); + my $dom = $hostdom{$perlvar{'lonHostID'}}; + my $internetdom = $intdom{$perlvar{'lonHostID'}}; + my %changes; + if (($dom ne '') && ($internetdom ne '')) { + if (keys(%hostdom)) { + my %currhosttypes; + if (open(my $fh,'<',"$perlvar{'lonTabDir'}/hosttypes.tab")) { + while (my $line = <$fh>) { + chomp($line); + my ($name,$value) = split(/:/,$line); + if (($name ne '') && ($value =~ /^(dom|intdom|other)$/)) { + $currhosttypes{$name} = $value; + } + } + close($fh); + } + if (open(my $fh,'>',"$perlvar{'lonTabDir'}/hosttypes.tab")) { + my $count = 0; + foreach my $lonid (sort(keys(%hostdom))) { + my $type = 'other'; + if ($hostdom{$lonid} eq $dom) { + $type = 'dom'; + } elsif ($intdom{$lonid} eq $internetdom) { + $type = 'intdom'; + } + print $fh "$lonid:$type\n"; + if (exists($currhosttypes{$lonid})) { + if ($type ne $currhosttypes{$lonid}) { + $changes{$lonid} = 1; + } + } else { + $changes{$lonid} = 1; + } + $count ++; + } + close($fh); + print "Completed writing host type data for $count hosts.\n"; + } + } else { + print "Writing of host types skipped - no hosts found.\n"; + } + } else { + print "Writing of host types skipped - could not determine this host's LON-CAPA domain or 'internet' domain.\n"; + } + return %changes; +} + +sub update_revocation_list { + my ($result,$changed) = &Apache::lonnet::fetch_crl_pemfile(); + if ($result eq 'ok') { + print "Certificate Revocation List (from CA) updated.\n"; + } else { + print "Certificate Revocation List from (CA) not updated.\n"; + } + return $changed; +} + +sub reset_nosslverify_pids { + my ($fh,%sslrem) = @_; + &checkon_daemon($fh,'lond',40000,'USR2'); + my $loncpidfile="$perlvar{'lonDaemons'}/logs/lonc.pid"; + my $loncppid; + if ((-e $loncpidfile) && (open(my $pfh,'<',$loncpidfile))) { + $loncppid=<$pfh>; + chomp($loncppid); + close($pfh); + if ($loncppid =~ /^\d+$/) { + my %pids_by_host; + my $docdir = $perlvar{'lonDocRoot'}; + if (-d "$docdir/lon-status/loncchld") { + if (opendir(my $dh,"$docdir/lon-status/loncchld")) { + while (my $file = readdir($dh)) { + next if ($file =~ /^\./); + if (open(my $fh,'<',"$docdir/lon-status/loncchld/$file")) { + my $record = <$fh>; + chomp($record); + close($fh); + my ($remotehost,$authmode) = split(/:/,$record); + $pids_by_host{$remotehost}{$authmode}{$file} = 1; + } + } + closedir($dh); + if (keys(%pids_by_host)) { + foreach my $host (keys(%pids_by_host)) { + if ($sslrem{$host}) { + if (ref($pids_by_host{$host}) eq 'HASH') { + if (ref($pids_by_host{$host}{'insecure'}) eq 'HASH') { + if (keys(%{$pids_by_host{$host}{'insecure'}})) { + foreach my $pid (keys(%{$pids_by_host{$host}{'insecure'}})) { + if (open(PIPE,"ps -o ppid= -p $pid |")) { + my $ppid = ; + chomp($ppid); + close(PIPE); + $ppid =~ s/(^\s+|\s+$)//g; + if (($ppid == $loncppid) && (kill 0 => $pid)) { + kill QUIT => $pid; + } + } + } + } + } + } + } + } + } + } + } + } + } + return; +} + sub send_mail { - print "Sending mail.\n"; my $defdom = $perlvar{'lonDefDomain'}; my $origmail = $perlvar{'lonAdmEMail'}; my $emailto = &Apache::loncommon::build_recipient_list(undef, @@ -790,11 +1095,34 @@ sub send_mail { if ($totalcount>2500) { $emailto.=",$perlvar{'lonSysEMail'}"; } - my $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; - - my $result=system("metasend -b -S 4000000 -t $emailto -s '$subj' -f $statusdir/index.html -m text/html >& /dev/null"); - if ($result != 0) { - $result=system("mail -s '$subj' $emailto < $statusdir/index.html"); + my $from; + my $hostname=`/bin/hostname`; + chop($hostname); + $hostname=~s/[^\w\.]//g; + if ($hostname) { + $from = 'www@'.$hostname; + } + my $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; + my $loncronmail = "To: $emailto\n". + "From: $from\n". + "Subject: ".$subj."\n". + "Content-type: text/html\; charset=UTF-8\n". + "MIME-Version: 1.0\n\n"; + if (open(my $fh,"<$statusdir/index.html")) { + while (<$fh>) { + $loncronmail .= $_; + } + close($fh); + } else { + $loncronmail .= "Failed to read from http://$hostname/lon-status/index.html\n"; + } + $loncronmail .= "\n\n"; + if (open(my $mailh, "|/usr/lib/sendmail -oi -t -odb")) { + print $mailh $loncronmail; + close($mailh); + print "Sending mail.\n"; + } else { + print "Sending mail failed.\n"; } } @@ -849,7 +1177,7 @@ sub main () { $hostname=~s/[^\w\.]//g; # make sure is safe to pass through shell my $subj="LON: Unconfigured machine $hostname"; system("echo 'Unconfigured machine $hostname.' |\ - mailto $emailto -s '$subj' > /dev/null"); + mail -s '$subj' $emailto > /dev/null"); exit 1; } @@ -860,14 +1188,28 @@ sub main () { my $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}"; my $subj="LON: $perlvar{'lonHostID'} User ID mismatch"; system("echo 'User ID mismatch. loncron must be run as user www.' |\ - mailto $emailto -s '$subj' > /dev/null"); + mail -s '$subj' $emailto > /dev/null"); exit 1; } # -------------------------------------------- Force reload of host information - &Apache::lonnet::load_hosts_tab(1); - &Apache::lonnet::load_domain_tab(1); - &Apache::lonnet::get_iphost(1); + my $nomemcache; + if ($justcheckdaemons) { + $nomemcache=1; + my $memcachepidfile="$perlvar{'lonDaemons'}/logs/memcached.pid"; + my $memcachepid; + if (-e $memcachepidfile) { + my $memfh=IO::File->new($memcachepidfile); + $memcachepid=<$memfh>; + chomp($memcachepid); + if ($memcachepid =~ /^\d+$/ && kill 0 => $memcachepid) { + undef($nomemcache); + } + } + } + &Apache::lonnet::load_hosts_tab(1,$nomemcache); + &Apache::lonnet::load_domain_tab(1,$nomemcache); + &Apache::lonnet::get_iphost(1,$nomemcache); # ----------------------------------------- Force firewall update for lond port @@ -904,6 +1246,7 @@ sub main () { &log_machine_info($fh); &clean_tmp($fh); &clean_lonIDs($fh); + &clean_webDAV_sessionIDs($fh); &check_httpd_logs($fh); &rotate_lonnet_logs($fh); &rotate_other_logs($fh); @@ -919,6 +1262,10 @@ sub main () { &checkon_daemon($fh,'lonr',40000); } if ($justreload) { + &clean_nosslverify($fh); + &write_connection_config(); + &write_hosttypes(); + &update_revocation_list(); &checkon_daemon($fh,'lond',40000,'USR2'); &checkon_daemon($fh,'lonc',40000,'USR2'); } @@ -927,11 +1274,20 @@ sub main () { } if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { &check_delayed_msg($fh); - &finish_logging($fh); &log_simplestatus(); &write_loncaparevs(); &write_serverhomeIDs(); &write_checksums(); + my %sslrem = &clean_nosslverify($fh); + my %conchgs = &write_connection_config(); + my %hosttypechgs = &write_hosttypes(); + my $hadcrlchg = &update_revocation_list(); + if ((keys(%conchgs) > 0) || (keys(%hosttypechgs) > 0) || + $hadcrlchg || (keys(%sslrem) > 0)) { + &checkon_daemon($fh,'lond',40000,'USR2'); + &reset_nosslverify_pids($fh,%sslrem); + } + &finish_logging($fh); if ($totalcount>200 && !$noemail) { &send_mail(); } } }