--- loncom/cgi/clusterstatus.pl	2003/07/31 19:55:37	1.12
+++ loncom/cgi/clusterstatus.pl	2003/08/05 18:33:08	1.16
@@ -3,10 +3,11 @@ $|=1;
 # The LearningOnline Network with CAPA
 # Cluster Status
 #
-# $Id: clusterstatus.pl,v 1.12 2003/07/31 19:55:37 www Exp $
+# $Id: clusterstatus.pl,v 1.16 2003/08/05 18:33:08 www Exp $
 
 use lib '/home/httpd/lib/perl/';
 use LONCAPA::Configuration;
+use strict;
 
 use LWP::UserAgent();
 use HTTP::Headers;
@@ -19,6 +20,28 @@ my %connectionstatus=();
 my %perlvar=();
 
 my $mode;
+my $concount=0;
+my $fromcache;
+
+my %domaindescription = ();
+my %domain_auth_def = ();
+my %domain_auth_arg_def = ();
+
+my %hostname=();
+my %hostip=();
+my %hostdom=();
+my %hostrole=();
+my %libserv=();
+
+my $maxusers=0;
+my $maxload=0;
+my $totalusers=0;
+
+my %FORM=();
+
+my $stat_total=0;
+my $stat_notyet=0;
+my $stat_fromcache=0;
 
 sub select_form {
     my ($def,$name,%hash) = @_;
@@ -47,13 +70,26 @@ sub hidden {
 
 sub request {
     my ($local,$url,$cachetime)=@_;
+    $cachetime*=(0.5+rand);
     my $key=&key($local,$url);
     my $reply='';
+    $stat_total++;
+# if fromcache flag is set, only return cached values
+    if ($fromcache) {
+	if ($FORM{$key.'_time'}) {
+	    return $FORM{$key};
+	    $stat_fromcache++;
+	} else {
+	    return 'not_yet';
+	    $stat_notyet++;
+	}
+    }
+# normal mode, refresh when expired or not yet present
     if ($FORM{$key.'_time'}) {
 	if ((time-$FORM{$key.'_time'})<$cachetime) {
 	    $reply=$FORM{$key};
 	    &hidden($key.'_time',$FORM{$key.'_time'});
-	    &hidden($key.'_fromcache',1);
+	    $stat_fromcache++;
 	}
     }
     unless ($reply) {
@@ -61,7 +97,7 @@ sub request {
 	    $reply='local_unknown'; 
 	} else {
 
-	    my $ua=new LWP::UserAgent(timeout => 20);
+	    my $ua=new LWP::UserAgent(timeout => 10);
     
 	    my $request=new HTTP::Request('GET',
 					  "http://".$hostname{$local}.$url);
@@ -91,10 +127,17 @@ sub connected {
     unless ($hostname{$remote}) { return 'remote_unknown'; }
     my $url='/cgi-bin/ping.pl?'.$remote;
 #
-# Slowly phase this in: if not cached, only do 10 percent of the cases 
+# Slowly phase this in: if not cached, only do 5 percent of the cases,
+# but always do the first five. 
 #
     unless ($FORM{&key($local,$url)}) {
-	unless (rand>0.9) { return 'not_yet'; }
+	unless (($concount<=5) || (rand>0.95)) {
+	    $stat_total++;
+	    $stat_notyet++; 
+	    return 'not_yet'; 
+	} else {
+	    $concount++;
+	}
     }
 #
 # Actually do the query
@@ -199,6 +242,10 @@ ENDHEADER
 	print ("<tr><td bgcolor='#DDBBBB'><font color='#552222' size='+2'>$trouble</font></td></tr>");
     }
     print "<tr><td bgcolor='#BBBBBB'>";
+# version
+    if ($host{$local.'_version'}) {
+	print "<br />Version: ".$host{$local.'_version'}
+    }
 # load
     if (($host{$local.'_load_doomed'}>0.5) || ($mode eq 'load_doomed')) {
 	print "<br />Load: ".$host{$local.'_load'}
@@ -249,125 +296,23 @@ sub doomedness {
     return sort { $alldoomed{$b} <=> $alldoomed{$a} } @allhosts;
 }
 
-# ====================================================================== Status
-sub statuslist {
-    my ($local,$what)=@_;
-    print 
-"<script>document.prgstat.progress.value='Testing $local ($hostname{$local}): $what';</script>\n";
-}
-
-#
-# Main program
-#
-# ========================================================= Get form parameters
-my $buffer;
-
-read(STDIN, $buffer, $ENV{'CONTENT_LENGTH'});
-my @pairs=split(/&/,$buffer);
-my $pair; my $name; my $value;
-undef %FORM;
-%FORM=();
-foreach $pair (@pairs) {
-    ($name,$value) = split(/=/,$pair);
-    $value =~ tr/+/ /;
-    $value =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C",hex($1))/eg;
-    $FORM{$name}=$value;
-} 
-
-$buffer=$ENV{'QUERY_STRING'};
-@pairs=split(/&/,$buffer);
-foreach $pair (@pairs) {
-    ($name,$value) = split(/=/,$pair);
-    $value =~ tr/+/ /;
-    $value =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C",hex($1))/eg;
-    $FORM{$name}=$value;
-} 
-
-# ====================================================== Determine refresh rate
-
-my $refresh=(($FORM{'refresh'}=~/^\d+$/)?$FORM{'refresh'}:120);
-if ($refresh<30) { $refresh=30; }
-my $starttime=time;
-
-# ============================================================== Determine mode
+sub resetvars {
+   $maxusers=0;
+   $maxload=0;
+   $totalusers=0;
+   $stat_total=0;
+   $stat_notyet=0;
+   $stat_fromcache=0;
 
-my %modes=('trouble' => 'Trouble',
-	   'users_doomed' => 'Doomed: Users',
-	   'loncron_doomed' => 'Doomed: General (loncron)',
-	   'mysql_doomed' => 'Doomed: Database (mysql)',
-	   'notconnected_doomed' => 'Doomed: Connections',
-	   'checkrpms_doomed' => 'Doomed: RPMs',
-	   'load_doomed' => 'Doomed: Load',
-	   'unresponsive_doomed' => 'Doomed: Status could not be determined',
-	   'users' => 'User Report',
-	   'load' => 'Load Report',
-	   'connections' => 'Connections Matrix');
-
-$mode=$FORM{'mode'};
-unless ($modes{$mode}) { $mode='trouble'; }
-# ================================================================ Send Headers
-print "Content-type: text/html\n\n".
-    "<html><body bgcolor='#FFFFFF'>\n";
-# -------------------- Read loncapa.conf (and by default, loncapa_apache.conf).
-my $perlvarref=LONCAPA::Configuration::read_conf('loncapa.conf');
-%perlvar=%{$perlvarref};
-undef $perlvarref; # remove since sensitive and not needed
-delete $perlvar{'lonReceipt'}; # remove since sensitive and not needed
-delete $perlvar{'lonSqlAccess'}; # remove since sensitive and not needed
-
-# ------------------------------------------------------------- Read hosts file
-{
-    my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab");
-
-    $total=0;
-    while (my $configline=<$config>) {
-       $configline=~s/#.*$//;
-       unless ($configline=~/\w/) { next; } 
-       my ($id,$domain,$role,$name,$ip)=split(/:/,$configline);
-       $hostname{$id}=$name;
-       $hostdom{$id}=$domain;
-       $hostrole{$id}=$role;
-       $hostip{$id}=$ip;
-       $total++;
-       if (($role eq 'library') && ($id ne $perlvar{'lonHostID'})) {
-	   $libserv{$id}=$name;
-       }
-    }
+   undef %host;
+   %host=();
 }
-# ------------------------------------------------------------ Read domain file
-{
-    my $fh=IO::File->new($perlvar{'lonTabDir'}.'/domain.tab');
-    %domaindescription = ();
-    %domain_auth_def = ();
-    %domain_auth_arg_def = ();
-    if ($fh) {
-       while (<$fh>) {
-           next if (/^(\#|\s*$)/);
-           chomp;
-           my ($domain, $domain_description, $def_auth, $def_auth_arg)
-               = split(/:/,$_,4);
-           $domain_auth_def{$domain}=$def_auth;
-           $domain_auth_arg_def{$domain}=$def_auth_arg;
-           $domaindescription{$domain}=$domain_description;
-       }
-    }
-}
-
-print "<img src='/adm/lonIcons/lonlogos.gif' align='right' /><h1>LON-CAPA Cluster Status ".localtime()."</h1>";
-print "<form name='prgstat'>\n".
-"<input type='text' name='progress' value='Starting ...' size='100' /><br />".
-"</form>\n";;
-print "<form name='status' method='post'>\n";
-print 'Choose next report: '.&select_form($mode,'mode',%modes).'<hr />';
-&hidden('refresh',$refresh);
 
+sub mainloop {
+    &resetvars();
 # ==================================================== Main Loop over all Hosts
 
-my $maxusers=0;
-my $maxload=0;
-my $totalusers=0;
-
-foreach $local (sort keys %hostname) {
+foreach my $local (sort keys %hostname) {
     $host{$local.'_unresponsive_doomed'}=0;
 # -- Check general status
     &statuslist($local,'General');
@@ -386,6 +331,15 @@ foreach $local (sort keys %hostname) {
 	    $host{$local.'_errors'}=$loncron{'errors'};
 	}
     }
+# -- Check version
+    &statuslist($local,'Version');
+    my $version=&request($local,'/lon-status/version.txt',7200);
+    if ($version eq 'local_error') {
+	$host{$local.'_version'}='Could not determine.';
+	$host{$local.'_unresponsive_doomed'}++;
+    } else {
+	$host{$local.'_version'}=$version;
+    }
 # -- Check user status
     &statuslist($local,'Users');
     my %userstatus=&replyhash($local,'/cgi-bin/userstatus.pl?simple',600);
@@ -452,7 +406,7 @@ foreach $local (sort keys %hostname) {
     &statuslist($local,'Connections');
     $host{$local.'_notconnected'}='';
     $host{$local.'_notconnected_doomed'}=0;
-    foreach $remote (sort keys %hostname) {
+    foreach my $remote (sort keys %hostname) {
 	my $status=&connected($local,$remote);
 	$connectionstatus{$local.'_TO_'.$remote}=$status;
 	unless (($status eq 'ok') || ($status eq 'not_yet')) {
@@ -460,9 +414,12 @@ foreach $local (sort keys %hostname) {
 	    $host{$local.'_notconnected_doomed'}++;
 	}
     }
-# =============================================================== End Mail Loop
+# =============================================================== End Main Loop
 }
-&statuslist('Done.');
+
+}
+
+sub reports {
 # ====================================================================== Output
     if ($mode=~/\_doomed$/) {
 # Output by doomedness
@@ -482,13 +439,27 @@ foreach $local (sort keys %hostname) {
 	    print '<tr><th bgcolor="#DDDDBB">'.$local.'</th>';
 	    foreach my $remote (sort keys %hostname) {
 		if ($connectionstatus{$local.'_TO_'.$remote} eq 'not_yet') {
-		    print '<td bgcolor="#FFFFFF">&nbsp;</td>';
+		    my $cellcolor='#FFFFFF';
+		    if ($local eq $remote) { $cellcolor='#DDDDDD'; }
+		    print '<td bgcolor="'.$cellcolor.'"><font color="#555522" size="-2">not yet tested</font></td>';
 		} elsif ($connectionstatus{$local.'_TO_'.$remote} eq 'ok') {
+		    my $cellcolor='#BBDDBB';
+		    if ($local eq $remote) { $cellcolor='#99DD99'; }
 		    print 
-'<td bgcolor="#BBDDBB"><font color="#225522" face="arial"><b>ok</b></td>';
+'<td bgcolor="'.$cellcolor.'"><font color="#225522" face="arial"><b>ok</b></td>';
 		} else {
+		    my $cellcolor='#DDBBBB';
+		    if ($connectionstatus{$local.'_TO_'.$remote} eq 'local_error') {
+			if ($local eq $remote) { 
+			    $cellcolor='#DD88AA'; 
+			} else {
+			    $cellcolor='#DDAACC';
+			}
+		    } else {
+			if ($local eq $remote) { $cellcolor='#DD9999'; }
+		    }
 		    print 
-		  '<td bgcolor="#DDBBBB"><font color="#552222" size="-2">'.
+		  '<td bgcolor="'.$cellcolor.'"><font color="#552222" size="-2">'.
 		  $connectionstatus{$local.'_TO_'.$remote}.'<br />';
 		    &lonc($local); &lond($remote);
 		    print '</td>';
@@ -504,11 +475,13 @@ foreach $local (sort keys %hostname) {
 	    print "<h3>Total active user(s): $totalusers</h3>". 
        "<table cellspacing='3' cellpadding='3' border='0' bgcolor='#225522'>";
 
-	    foreach $local (sort keys %hostname) {
+	    foreach my $local (sort keys %hostname) {
 		if (defined($host{$local.'_users'})) {
 		    print 
-'<tr><th bgcolor="#BBDDBB"><font face="arial" color="#225522" size="+1">'.$local.
-			'</font></th><td bgcolor="#DDDDBB">';
+'<tr><td bgcolor="#BBDDBB"><font face="arial" color="#225522" size="+1">'.$local.
+			'</font><br /><font size="-2">'.
+			$domaindescription{$hostdom{$local}}.
+		       '</font></td><td bgcolor="#DDDDBB">';
 		    &users($local);
 		    print 
 	      '</td><td bgcolor="#DDDDBB"><font face="arial" color="#225522">'.
@@ -527,12 +500,14 @@ foreach $local (sort keys %hostname) {
 	    my $factor=50/$maxload; 
 	    print
        "<table cellspacing='3' cellpadding='3' border='0' bgcolor='#225522'>";
-	    foreach $local (sort keys %hostname) {
+	    foreach my $local (sort keys %hostname) {
 		if (defined($host{$local.'_load_doomed'})) {
 		    print 
-'<tr><th bgcolor="#BBDDBB"><font face="arial" color="#225522" size="+1">'.
+'<tr><td bgcolor="#BBDDBB"><font face="arial" color="#225522" size="+1">'.
                         $local.
-			'</font></th><td bgcolor="#DDDDBB">';
+			'</font><br /><font size="-2">'.
+			$domaindescription{$hostdom{$local}}.
+		       '</font></td><td bgcolor="#DDDDBB">';
 		    &server($local);
 		    print 
 	      '</td><td bgcolor="#DDDDBB"><font face="arial" color="#225522">'.
@@ -547,8 +522,11 @@ foreach $local (sort keys %hostname) {
 	}
     } elsif ($mode eq 'trouble') {
 	my $count=0;
-	foreach $local (sort keys %hostname) {
+	foreach my $local (sort keys %hostname) {
 	    my $trouble='';
+	    if ($host{$local.'_unresponsive_doomed'}>3) {
+		$trouble='Does not respond to several queries.<br />';
+	    }
 	    if ($host{$local.'_errors'}) {
 		$trouble='Has loncron errors.<br />';
 	    } elsif ($host{$local.'_loncron_doomed'}>600) {
@@ -570,13 +548,145 @@ foreach $local (sort keys %hostname) {
 	}
 	unless ($count) { print "No mayor trouble."; }
     }
+}
+
+# ====================================================================== Status
+sub statuslist {
+    my ($local,$what)=@_;
+    print 
+"<script>document.prgstat.progress.value='Testing $local ($hostname{$local}): $what';</script>\n";
+}
+
+# =============================================================================
+# =============================================================================
+# Main program
+#
+# ========================================================= Get form parameters
+my $buffer;
+
+read(STDIN, $buffer, $ENV{'CONTENT_LENGTH'});
+my @pairs=split(/&/,$buffer);
+my $pair; my $name; my $value;
+undef %FORM;
+%FORM=();
+foreach $pair (@pairs) {
+    ($name,$value) = split(/=/,$pair);
+    $value =~ tr/+/ /;
+    $value =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C",hex($1))/eg;
+    $FORM{$name}=$value;
+} 
+
+$buffer=$ENV{'QUERY_STRING'};
+@pairs=split(/&/,$buffer);
+foreach $pair (@pairs) {
+    ($name,$value) = split(/=/,$pair);
+    $value =~ tr/+/ /;
+    $value =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C",hex($1))/eg;
+    $FORM{$name}=$value;
+} 
+
+# ====================================================== Determine refresh rate
+
+my $refresh=(($FORM{'refresh'}=~/^\d+$/)?$FORM{'refresh'}:30);
+if ($refresh<30) { $refresh=30; }
+my $starttime=time;
+
+# ============================================================== Determine mode
+
+my %modes=('trouble' => 'Trouble',
+	   'users_doomed' => 'Doomed: Users',
+	   'loncron_doomed' => 'Doomed: General (loncron)',
+	   'mysql_doomed' => 'Doomed: Database (mysql)',
+	   'notconnected_doomed' => 'Doomed: Connections',
+	   'checkrpms_doomed' => 'Doomed: RPMs',
+	   'load_doomed' => 'Doomed: Load',
+	   'unresponsive_doomed' => 'Doomed: Status could not be determined',
+	   'users' => 'User Report',
+	   'load' => 'Load Report',
+	   'connections' => 'Connections Matrix');
+
+$mode=$FORM{'mode'};
+unless ($modes{$mode}) { $mode='trouble'; }
+# ================================================================ Send Headers
+print "Content-type: text/html\n\n".
+    "<html><body bgcolor='#FFFFFF'>\n";
+# -------------------- Read loncapa.conf (and by default, loncapa_apache.conf).
+my $perlvarref=LONCAPA::Configuration::read_conf('loncapa.conf');
+%perlvar=%{$perlvarref};
+undef $perlvarref; # remove since sensitive and not needed
+delete $perlvar{'lonReceipt'}; # remove since sensitive and not needed
+delete $perlvar{'lonSqlAccess'}; # remove since sensitive and not needed
+
+# ------------------------------------------------------------- Read hosts file
+{
+    my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab");
+
+    while (my $configline=<$config>) {
+       $configline=~s/#.*$//;
+       unless ($configline=~/\w/) { next; } 
+       my ($id,$domain,$role,$name,$ip)=split(/:/,$configline);
+       $hostname{$id}=$name;
+       $hostdom{$id}=$domain;
+       $hostrole{$id}=$role;
+       $hostip{$id}=$ip;
+       if (($role eq 'library') && ($id ne $perlvar{'lonHostID'})) {
+	   $libserv{$id}=$name;
+       }
+    }
+}
+# ------------------------------------------------------------ Read domain file
+{
+    my $fh=IO::File->new($perlvar{'lonTabDir'}.'/domain.tab');
+    if ($fh) {
+       while (<$fh>) {
+           next if (/^(\#|\s*$)/);
+           chomp;
+           my ($domain, $domain_description, $def_auth, $def_auth_arg)
+               = split(/:/,$_,4);
+           $domain_auth_def{$domain}=$def_auth;
+           $domain_auth_arg_def{$domain}=$def_auth_arg;
+           $domaindescription{$domain}=$domain_description;
+       }
+    }
+}
+
+print "<img src='/adm/lonIcons/lonlogos.gif' align='right' /><h1>LON-CAPA Cluster Status ".localtime()."</h1>";
+print "<form name='prgstat'>\n".
+"<input type='text' name='progress' value='Starting ...' size='100' /><br />".
+"</form>\n";;
+print "<form name='status' method='post'>\n";
+print 'Choose next report: '.&select_form($mode,'mode',%modes).'<hr />';
+&hidden('refresh',$refresh);
+
+    if (!$FORM{'runonetime'}) {
+	print 
+   "<h3>Gathering initial cluster data</h3>This may take some time ...<br />";
+	$fromcache=0;
+	&mainloop();
+	&statuslist('Done initial run.');
+	&reports();
+    } else {
+	$fromcache=1;
+	&mainloop();
+	&statuslist('Done gathering cached data');
+	&reports();
+	$fromcache=0;
+	&mainloop();
+    }
+    &hidden('runonetime',1);
+print '<tt><br />Total number of queries: '.$stat_total.
+          '<br />Percent complete: '.
+	  int(($stat_total-$stat_notyet)/$stat_total*100.).
+	  '<br />Percent from cache: '.
+          int($stat_fromcache/$stat_total*100.).'</tt>';
+
 # ============================================================== Close, refresh
 print "</form><script>";
-$runtime=time-$starttime;
-if ($runtime>=$refresh) {
-    print 'document.status.submit();';
+my $runtime=time-$starttime;
+if (($refresh-$runtime)<0) {
+    print "document.status.submit();";
 } else {
-    $refreshtime=int(1000*($refresh-$runtime));
+    my $refreshtime=int(1000*($refresh-$runtime));
     print "setTimeout('document.status.submit()',$refreshtime);\n".
           "document.prgstat.progress.value='Will automatically refresh ($refresh secs refresh cycle)'";
 }