--- loncom/cgi/clusterstatus.pl	2002/05/11 21:29:42	1.3
+++ loncom/cgi/clusterstatus.pl	2003/07/31 15:35:02	1.9
@@ -2,11 +2,8 @@
 $|=1;
 # The LearningOnline Network with CAPA
 # Cluster Status
-# (Versions
-# (Running loncron
-# 09/06/01 Gerd Kortemeyer)
-# 02/18/02,02/19/02 Gerd Kortemeyer)
-# 5/11/2002 Scott Harrison
+#
+# $Id: clusterstatus.pl,v 1.9 2003/07/31 15:35:02 www Exp $
 
 use lib '/home/httpd/lib/perl/';
 use LONCAPA::Configuration;
@@ -14,49 +11,276 @@ use LONCAPA::Configuration;
 use LWP::UserAgent();
 use HTTP::Headers;
 use IO::File;
-use Net::Ping;
 
-sub online {
-    my $host=shift;
-    my $p=Net::Ping->new("tcp",10);
-    my $online=$p->ping("$host");
-    $p->close();
-    undef ($p);
-    return $online;
+my %host=();
+my $oneday=60*60*24;
+
+my %connectionstatus=();
+my %perlvar=();
+
+my $mode;
+
+sub select_form {
+    my ($def,$name,%hash) = @_;
+    my $selectform = "<select name=\"$name\" size=\"1\">\n";
+    foreach (sort keys %hash) {
+        $selectform.="<option value=\"$_\" ".
+            ($_ eq $def ? 'selected' : '').
+                ">".$hash{$_}."</option>\n";
+    }
+    $selectform.="</select>";
+    return $selectform;
+}
+
+
+sub key {
+    my ($local,$url)=@_;
+    my $key=$local.'_'.$url;
+    $key=~s/\W/\_/gs;
+    return $key;
 }
 
+sub hidden {
+    my ($name,$value)=@_;
+    print "\n<input type='hidden' name='$name' value='$value' />";
+}
+
+sub request {
+    my ($local,$url,$cachetime)=@_;
+    my $key=&key($local,$url);
+    my $reply='';
+    if ($FORM{$key.'_time'}) {
+	if ((time-$FORM{$key.'_time'})<$cachetime) {
+	    $reply=$FORM{$key};
+	    &hidden($key.'_time',$FORM{$key.'_time'});
+	    &hidden($key.'_fromcache',1);
+	}
+    }
+    unless ($reply) {
+	unless ($hostname{$local}) { 
+	    $reply='local_unknown'; 
+	} else {
+
+	    my $ua=new LWP::UserAgent(timeout => 20);
+    
+	    my $request=new HTTP::Request('GET',
+					  "http://".$hostname{$local}.$url);
+	    $request->authorization_basic('lonadm','litelite');
+
+	    my $response=$ua->request($request);
+
+	    unless ($response->is_success) { 
+		$reply='local_error'; 
+	    } else {
+		$reply=$response->content;
+		chomp($reply);
+	    }
+	}
+	&hidden($key.'_time',time);
+    }
+    &hidden($key,$reply);
+    return $reply;
+}
+
+# ============================================= Are local and remote connected?
 sub connected {
     my ($local,$remote)=@_;
     $local=~s/\W//g;
     $remote=~s/\W//g;
 
-    unless ($hostname{$local}) { return 'local_unknown'; }
     unless ($hostname{$remote}) { return 'remote_unknown'; }
+    my $url='/cgi-bin/ping.pl?'.$remote;
+#
+# Slowly phase this in: if not cached, only do 10 percent of the cases 
+#
+    unless ($FORM{&key($local,$url)}) {
+	unless (rand>0.9) { return 'not_yet'; }
+    }
+#
+# Actually do the query
+#
+    &statuslist($local,'connecting '.$remote);
+    my $reply=&request($local,$url,3600);
+    $reply=(split("\n",$reply))[0];
+    $reply=~s/\W//g;
+    if ($reply ne $remote) { return $reply; }
+    return 'ok';
+}
+# ============================================================ Get a reply hash
 
-    unless (&online($hostname{$local})) { return 'local_offline'; }
+sub replyhash {
+    my %returnhash=();
+    foreach (split(/\&/,&request(@_))) {
+	my ($name,$value)=split(/\=/,$_);
+	if ($name) {
+	    unless ($value) { $value=''; }
+	    $returnhash{$name}=$value;
+	}
+    }
+    return %returnhash;
+}
 
-    my $ua=new LWP::UserAgent;
-    
-    my $request=new HTTP::Request('GET',
-      "http://".$hostname{$local}.'/cgi-bin/ping.pl?'.$remote);
+# ================================================================ Link to host
 
-    my $response=$ua->request($request);
+sub otherwindow {
+    my ($local,$url,$label)=@_;
+    return
+  " <a href='http://$hostname{$local}$url' target='newwin$local'>$label</a> ";
+}
 
-    unless ($response->is_success) { return 'local_error'; }
+sub login {
+    my $local=shift;
+    print &otherwindow($local,'/adm/login?domain='.$perlvar{'lonDefDomain'},
+		       'Login');
+}
 
-    my $reply=$response->content;
-    $reply=(split("\n",$reply))[0];
-    $reply=~s/\W//g;
-    if ($reply ne $remote) { return $reply; }
-    return 'ok';
+sub runloncron {
+    my $local=shift;
+    print &otherwindow($local,'/cgi-bin/loncron.pl','Run loncron');
 }
 
- 
+sub loncron {
+    my $local=shift;
+    print &otherwindow($local,'/lon-status','loncron');
+}
+
+sub lonc {
+    my $local=shift;
+    print &otherwindow($local,'/lon-status/loncstatus.txt','lonc');
+}
+
+sub lond {
+    my $local=shift;
+    print &otherwindow($local,'/lon-status/londstatus.txt','lond');
+}
+
+sub users {
+    my $local=shift;
+    print &otherwindow($local,'/cgi-bin/userstatus.pl','Users');
+}
+
+sub versions {
+    my $local=shift;
+    print &otherwindow($local,'/cgi-bin/lonversions.pl','Versions');
+}
+
+sub server {
+    my $local=shift;
+    print &otherwindow($local,'/server-status','Server Status');
+}
+
+# ========================================================== Show server status
+
+sub serverstatus {
+    my $local=shift;
+    print (<<ENDHEADER);
+<table width="100%" bgcolor="#225522" cellspacing="2" cellpadding="2" border="0">
+<tr><td bgcolor="#BBDDBB"><font color="#225522" face="arial"><b>
+$local $hostdom{$local}</b> <tt>($hostname{$local}; $hostrole{$local})</tt>
+<br />$domaindescription{$hostdom{$local}}
+</font></th></tr><tr><td bgcolor="DDDDBB"><font color="#225522">
+ENDHEADER
+    &login($local);&server($local);&users($local);&versions($local);
+    &loncron($local);&lond($local);&lonc($local);&runloncron($local);
+    print "</font></td></tr><tr><td bgcolor='#BBBBBB'>";
+# load
+    if (($host{$local.'_load_doomed'}>0.5) || ($mode eq 'load_doomed')) {
+	print "<br />Load: ".$host{$local.'_load'}
+    }
+# users
+    if (($host{$local.'_users_doomed'}>10) || ($mode eq 'users_doomed')) {
+	print "<br />Active Users: ".$host{$local.'_users'}
+    }
+
+# checkrpms
+    if ($host{$local.'_checkrpms'}) {
+	print "<br />RPMs: ".$host{$local.'_checkrpms'}
+    }
+# mysql
+    if ($host{$local.'_mysql'}) {
+	print "<br />MySQL Database: ".$host{$local.'_mysql'}
+    }
+    print "</td></tr></table><br />";
+}
+
+# =========================================================== Doomedness sorted
+
+sub doomedness {
+    my $crit=shift;
+    my %alldoomed=();
+    my @allhosts=();
+    foreach (keys %host) {
+	if ($_=~/^(\w+)\_$crit$/) {
+	    if ($host{$_}) {
+		push (@allhosts,$1);
+		$alldoomed{$1}=$host{$_};
+	    }
+	}
+    }
+    return sort { $alldoomed{$b} <=> $alldoomed{$a} } @allhosts;
+}
+
+# ====================================================================== Status
+sub statuslist {
+    my ($local,$what)=@_;
+    print 
+"<script>document.prgstat.progress.value='Testing $local ($hostname{$local}): $what';</script>\n";
+}
+
+#
+# Main program
+#
+# ========================================================= Get form parameters
+my $buffer;
+
+read(STDIN, $buffer, $ENV{'CONTENT_LENGTH'});
+my @pairs=split(/&/,$buffer);
+my $pair; my $name; my $value;
+undef %FORM;
+%FORM=();
+foreach $pair (@pairs) {
+    ($name,$value) = split(/=/,$pair);
+    $value =~ tr/+/ /;
+    $value =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C",hex($1))/eg;
+    $FORM{$name}=$value;
+} 
+
+$buffer=$ENV{'QUERY_STRING'};
+@pairs=split(/&/,$buffer);
+foreach $pair (@pairs) {
+    ($name,$value) = split(/=/,$pair);
+    $value =~ tr/+/ /;
+    $value =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C",hex($1))/eg;
+    $FORM{$name}=$value;
+} 
+
+# ====================================================== Determine refresh rate
+
+my $refresh=(($FORM{'refresh'}=~/^\d+$/)?$FORM{'refresh'}:60);
+if ($refresh<30) { $refresh=30; }
+my $starttime=time;
+
+# ============================================================== Determine mode
+
+my %modes=('trouble' => 'Trouble',
+	   'users_doomed' => 'Doomed: Users',
+	   'loncron_doomed' => 'Doomed: General (loncron)',
+	   'mysql_doomed' => 'Doomed: Database (mysql)',
+	   'notconnected_doomed' => 'Doomed: Connections',
+	   'checkrpms_doomed' => 'Doomed: RPMs',
+	   'load_doomed' => 'Doomed: Load',
+	   'unresponsive_doomed' => 'Doomed: Status could not be determined',
+	   'users' => 'User Report',
+	   'connections' => 'Connections Matrix');
+
+$mode=$FORM{'mode'};
+unless ($modes{$mode}) { $mode='trouble'; }
+# ================================================================ Send Headers
 print "Content-type: text/html\n\n".
-      "<html><body bgcolor=#FFFFFF>\n";
-# ------------------------------------------- Read access.conf and loncapa.conf
-my $perlvarref=LONCAPA::Configuration::read_conf('access.conf','loncapa.conf');
-my %perlvar=%{$perlvarref};
+    "<html><body bgcolor=#FFFFFF>\n";
+# -------------------- Read loncapa.conf (and by default, loncapa_apache.conf).
+my $perlvarref=LONCAPA::Configuration::read_conf('loncapa.conf');
+%perlvar=%{$perlvarref};
 undef $perlvarref; # remove since sensitive and not needed
 delete $perlvar{'lonReceipt'}; # remove since sensitive and not needed
 delete $perlvar{'lonSqlAccess'}; # remove since sensitive and not needed
@@ -67,6 +291,8 @@ delete $perlvar{'lonSqlAccess'}; # remov
 
     $total=0;
     while (my $configline=<$config>) {
+       $configline=~s/#.*$//;
+       unless ($configline=~/\w/) { next; } 
        my ($id,$domain,$role,$name,$ip)=split(/:/,$configline);
        $hostname{$id}=$name;
        $hostdom{$id}=$domain;
@@ -78,78 +304,135 @@ delete $perlvar{'lonSqlAccess'}; # remov
        }
     }
 }
-
-print 
-"<h1>Cluster Status</h1>";
-print "Please be patient while building cluster status ...<p>\n";
-$table='';
-
-$table.="<table border=2><tr><th rowspan=2>From (local)</th>".
-"<th colspan=$total>To (remote)</th></tr><tr>";
-foreach $remote (sort keys %hostname) {
-   $table.="<th";
-   if ($remote eq $perlvar{'lonHostID'}) {
-       $table.=" bgcolor=#CCFFBB";
-   } else {
-       if ($libserv{$remote}) { $table.=" bgcolor=#FFFF99"; }
-   }
-   $table.=
- "><a href=http://".$hostname{$remote}."/lon-status/>$remote</a></th>";
+# ------------------------------------------------------------ Read domain file
+{
+    my $fh=IO::File->new($perlvar{'lonTabDir'}.'/domain.tab');
+    %domaindescription = ();
+    %domain_auth_def = ();
+    %domain_auth_arg_def = ();
+    if ($fh) {
+       while (<$fh>) {
+           next if (/^(\#|\s*$)/);
+           chomp;
+           my ($domain, $domain_description, $def_auth, $def_auth_arg)
+               = split(/:/,$_,4);
+           $domain_auth_def{$domain}=$def_auth;
+           $domain_auth_arg_def{$domain}=$def_auth_arg;
+           $domaindescription{$domain}=$domain_description;
+       }
+    }
 }
-$table.="</tr>";
+
+
+print "<h1>Cluster Status ".localtime()."</h1>";
+print "<form name='prgstat'>\n".
+"<input type='text' name='progress' value='Starting ...' size='100' /><br />".
+"</form>\n";;
+print "<form name='status' method='post'>\n";
+print 'Choose next report: '.&select_form($mode,'mode',%modes).'<hr />';
+&hidden('refresh',$refresh);
+
+# ==================================================== Main Loop over all Hosts
+
 foreach $local (sort keys %hostname) {
-    print "Checking $local ";
-   $table.="<tr><td";
-   if ($local eq $perlvar{'lonHostID'}) {
-       $table.=" bgcolor=#CCFFBB";
-   } else {
-       if ($libserv{$local}) { $table.=" bgcolor=#FFFF99"; }
-   }
-   $table.=
- "><b><a href=http://".$hostname{$local}."/lon-status/>$local</a></b><br>".
-     $hostrole{$local}.' '.$hostdom{$local}.'<br>'.
-     "<tt><a href=http://".$hostname{$local}."/server-status/>".
-      $hostname{$local}."</a></tt><br>".
-     "<a href=http://".$hostname{$local}."/cgi-bin/loncron.pl>New Report</a>".
-     "</td>";
-  if (&online($hostname{$local})) {
-   foreach $remote (sort keys %hostname) {
-       $status=&connected($local,$remote);
-       if ($status eq 'ok') {
-	   $table.="<td";
-           if ($libserv{$remote}) {
-	       if ($libserv{$local}) {
-		   $table.=" bgcolor=#FFFF44";
-               } else {
-                   $table.=" bgcolor=#FFFF99";
-               }
-           } elsif ($libserv{$local}) {
-               $table.=" bgcolor=#FFFF99";
-           }
-           $table.="><b><font color=#00FF00>OK</font></b></td>";
-           print ".";
-       } else {
-           print "X";
-           $table.="<td><b><font color=#FF0000>".$status."</font></b><br>".
-     "<a href=http://".$hostname{$local}.
-	 "/lon-status/loncstatus.txt>lonc</a>";
-           if (&online($hostname{$remote})) {
-              $table.=" <a href=http://".$hostname{$remote}.
-      "/lon-status/londstatus.txt>lond</a>".
-            "</td>";
-	  } else {
-              $table.=' offline';
-          }
-  
-       }
-   }
-  } else {
-      print "offline";
-      $table.='<th colspan='.$total.'><font color=#FF0000>Offline</font></th>';
-  }
-   $table.="</tr>\n";
-    print "<br>\n";
-}
-$table.="</table>";
-print $table;
-print "</body></html>";
+    $host{$local.'_unresponsive_doomed'}=0;
+# -- Check general status
+    &statuslist($local,'General');
+    my %loncron=&replyhash($local,'/lon-status/loncron_simple.txt',1200);
+    if (defined($loncron{'local_error'})) {
+	$host{$local.'_loncron'}='Could not determine.';
+	$host{$local.'_unresponsive_doomed'}++;
+    } else {
+	if ((time-$loncron{'time'})>$oneday) {
+	    $host{$local.'_loncron'}='Stale.';
+	    $host{$local.'_unresponsive_doomed'}++;
+	} else {
+	}
+    }
+# -- Check user status
+    &statuslist($local,'Users');
+    my %userstatus=&replyhash($local,'/cgi-bin/userstatus.pl?simple',600);
+    if (defined($userstatus{'local_error'})) {
+	$host{$local.'_userstatus'}='Could not determine.';
+	$host{$local.'_unresponsive_doomed'}++;
+    } else {
+	$host{$local.'_users_doomed'}=$userstatus{'Active'};
+	$host{$local.'_users'}=$userstatus{'Active'};
+	my ($sload,$mload,$lload)=split(/ /,$userstatus{'loadavg'});
+	$host{$local.'_load_doomed'}=$mload;
+	$host{$local.'_load'}=$userstatus{'loadavg'};
+    }
+# -- Check mysql status
+    &statuslist($local,'Database');
+    my %mysql=&replyhash($local,'/lon-status/mysql.txt',3600);
+    if (defined($mysql{'local_error'})) {
+	$host{$local.'_mysql'}='Could not determine.';
+	$host{$local.'_unresponsive_doomed'}++;
+    } else {
+	if ((time-$mysql{'time'})>(7*$oneday)) {
+	    if ($hostrole{$local} eq 'library') {
+		$host{$local.'_mysql'}='Stale.';
+		$host{$local.'_mysql_doomed'}=1;
+	    }
+	    if ($mysql{'mysql'} eq 'defunct') {
+		$host{$local.'_mysql'}='Defunct (maybe stale).';
+		$host{$local.'_mysql_doomed'}=2;
+	    }
+	} elsif ($mysql{'mysql'} eq 'defunct') {
+	    $host{$local.'_mysql'}='Defunct.';
+	    $host{$local.'_mysql_doomed'}=3;
+	}
+    }
+# -- Check rpm status
+    &statuslist($local,'RPMs');
+    my %checkrpms=&replyhash($local,'/lon-status/checkrpms.txt',7200);
+    if (defined($checkrpms{'local_error'})) {
+	$host{$local.'_checkrpms'}='Could not determine.';
+	$host{$local.'_unresponsive_doomed'}++;
+    } else {
+	if ((time-$checkrpms{'time'})>(4*$oneday)) {
+	    $host{$local.'_checkrpms'}='Stale.';
+	    $host{$local.'_checkrpms_doomed'}=50;
+	    $host{$local.'_unresponsive_doomed'}++;
+	} elsif ($checkrpms{'status'} eq 'fail') {
+	    $host{$local.'_checkrpms'}='Could not checked RPMs.';
+	    $host{$local.'_checkrpms_doomed'}=100;
+	} elsif ($checkrpms{'rpmcount'}) {
+	    $host{$local.'_checkrpms'}='Outdated RPMs: '.
+		$checkrpms{'rpmcount'};
+	    $host{$local.'_checkrpms_doomed'}=$checkrpms{'rpmcount'};
+	}
+    }
+# -- Check connections
+    &statuslist($local,'Connections');
+    $host{$local.'_notconnected'}='';
+    $host{$local.'_notconnected_doomed'}=0;
+    foreach $remote (sort keys %hostname) {
+	my $status=&connected($local,$remote);
+	$connectionstatus{$local.'_TO_'.$remote}=$status;
+	unless (($status eq 'ok') || ($status eq 'not_yet')) {
+	    $host{$local.'_notconnected'}.=' '.$remote;
+	    $host{$local.'_notconnected_doomed'}++;
+	}
+    }
+# =============================================================== End Mail Loop
+}
+&statuslist('Done.');
+# ====================================================================== Output
+    if ($mode=~/\_doomed$/) {
+# Output by doomedness
+	foreach (&doomedness($mode)) {
+	    &serverstatus($_);
+	}
+    }
+# ============================================================== Close, refresh
+print "</form><script>";
+$runtime=time-$starttime;
+if ($runtime>=$refresh) {
+    print 'document.status.submit();';
+} else {
+    $refreshtime=int(1000*($refresh-$runtime));
+    print "setTimeout('document.status.submit()',$refreshtime);";
+}
+print "</script></body></html>";
+exit 0;