--- loncom/cgi/clusterstatus.pl	2003/07/30 21:20:16	1.8
+++ loncom/cgi/clusterstatus.pl	2003/07/31 16:07:47	1.10
@@ -2,10 +2,8 @@
 $|=1;
 # The LearningOnline Network with CAPA
 # Cluster Status
-# (Versions
-# (Running loncron
-# 09/06/01 Gerd Kortemeyer)
-# 02/18/02,02/19/02 Gerd Kortemeyer)
+#
+# $Id: clusterstatus.pl,v 1.10 2003/07/31 16:07:47 www Exp $
 
 use lib '/home/httpd/lib/perl/';
 use LONCAPA::Configuration;
@@ -18,6 +16,22 @@ my %host=();
 my $oneday=60*60*24;
 
 my %connectionstatus=();
+my %perlvar=();
+
+my $mode;
+
+sub select_form {
+    my ($def,$name,%hash) = @_;
+    my $selectform = "<select name=\"$name\" size=\"1\">\n";
+    foreach (sort keys %hash) {
+        $selectform.="<option value=\"$_\" ".
+            ($_ eq $def ? 'selected' : '').
+                ">".$hash{$_}."</option>\n";
+    }
+    $selectform.="</select>";
+    return $selectform;
+}
+
 
 sub key {
     my ($local,$url)=@_;
@@ -86,7 +100,7 @@ sub connected {
 # Actually do the query
 #
     &statuslist($local,'connecting '.$remote);
-    my $reply=&request($local,$url,1800);
+    my $reply=&request($local,$url,3600);
     $reply=(split("\n",$reply))[0];
     $reply=~s/\W//g;
     if ($reply ne $remote) { return $reply; }
@@ -106,17 +120,78 @@ sub replyhash {
     return %returnhash;
 }
 
-# ========================================================== Show server status
+# ================================================================ Link to host
 
 sub otherwindow {
     my ($local,$url,$label)=@_;
     return
-  "<a href='http://$hostname{$local}$url' target='newwin$local'>$label</a>";
+  " <a href='http://$hostname{$local}$url' target='newwin$local'>$label</a> ";
 }
 
+sub login {
+    my $local=shift;
+    print &otherwindow($local,'/adm/login?domain='.$perlvar{'lonDefDomain'},
+		       'Login');
+}
+
+sub runloncron {
+    my $local=shift;
+    print &otherwindow($local,'/cgi-bin/loncron.pl','Run loncron');
+}
+
+sub loncron {
+    my $local=shift;
+    print &otherwindow($local,'/lon-status','loncron');
+}
+
+sub lonc {
+    my $local=shift;
+    print &otherwindow($local,'/lon-status/loncstatus.txt','lonc');
+}
+
+sub lond {
+    my $local=shift;
+    print &otherwindow($local,'/lon-status/londstatus.txt','lond');
+}
+
+sub users {
+    my $local=shift;
+    print &otherwindow($local,'/cgi-bin/userstatus.pl','Users');
+}
+
+sub versions {
+    my $local=shift;
+    print &otherwindow($local,'/cgi-bin/lonversions.pl','Versions');
+}
+
+sub server {
+    my $local=shift;
+    print &otherwindow($local,'/server-status','Server Status');
+}
+
+# ========================================================== Show server status
+
 sub serverstatus {
     my $local=shift;
-    print "\n<hr /><h3>$local $hostdom{$local} ($hostname{$local}; $hostrole{$local})</h3>\n";
+    print (<<ENDHEADER);
+<table width="100%" bgcolor="#225522" cellspacing="2" cellpadding="2" border="0">
+<tr><td bgcolor="#BBDDBB"><font color="#225522" face="arial"><b>
+$local $hostdom{$local}</b> <tt>($hostname{$local}; $hostrole{$local})</tt>
+<br />$domaindescription{$hostdom{$local}}
+</font></th></tr><tr><td bgcolor="#DDDDBB"><font color="#225522">
+ENDHEADER
+    &login($local);&server($local);&users($local);&versions($local);
+    &loncron($local);&lond($local);&lonc($local);&runloncron($local);
+    print "</font></td></tr><tr><td bgcolor='#BBBBBB'>";
+# load
+    if (($host{$local.'_load_doomed'}>0.5) || ($mode eq 'load_doomed')) {
+	print "<br />Load: ".$host{$local.'_load'}
+    }
+# users
+    if (($host{$local.'_users_doomed'}>10) || ($mode eq 'users_doomed')) {
+	print "<br />Active Users: ".$host{$local.'_users'}
+    }
+
 # checkrpms
     if ($host{$local.'_checkrpms'}) {
 	print "<br />RPMs: ".$host{$local.'_checkrpms'}
@@ -125,6 +200,24 @@ sub serverstatus {
     if ($host{$local.'_mysql'}) {
 	print "<br />MySQL Database: ".$host{$local.'_mysql'}
     }
+    print "</td></tr></table><br />";
+}
+
+# =========================================================== Doomedness sorted
+
+sub doomedness {
+    my $crit=shift;
+    my %alldoomed=();
+    my @allhosts=();
+    foreach (keys %host) {
+	if ($_=~/^(\w+)\_$crit$/) {
+	    if ($host{$_}) {
+		push (@allhosts,$1);
+		$alldoomed{$1}=$host{$_};
+	    }
+	}
+    }
+    return sort { $alldoomed{$b} <=> $alldoomed{$a} } @allhosts;
 }
 
 # ====================================================================== Status
@@ -166,12 +259,28 @@ foreach $pair (@pairs) {
 my $refresh=(($FORM{'refresh'}=~/^\d+$/)?$FORM{'refresh'}:60);
 if ($refresh<30) { $refresh=30; }
 my $starttime=time;
+
+# ============================================================== Determine mode
+
+my %modes=('trouble' => 'Trouble',
+	   'users_doomed' => 'Doomed: Users',
+	   'loncron_doomed' => 'Doomed: General (loncron)',
+	   'mysql_doomed' => 'Doomed: Database (mysql)',
+	   'notconnected_doomed' => 'Doomed: Connections',
+	   'checkrpms_doomed' => 'Doomed: RPMs',
+	   'load_doomed' => 'Doomed: Load',
+	   'unresponsive_doomed' => 'Doomed: Status could not be determined',
+	   'users' => 'User Report',
+	   'connections' => 'Connections Matrix');
+
+$mode=$FORM{'mode'};
+unless ($modes{$mode}) { $mode='trouble'; }
 # ================================================================ Send Headers
 print "Content-type: text/html\n\n".
-    "<html><body bgcolor=#FFFFFF>\n";
+    "<html><body bgcolor='#FFFFFF'>\n";
 # -------------------- Read loncapa.conf (and by default, loncapa_apache.conf).
 my $perlvarref=LONCAPA::Configuration::read_conf('loncapa.conf');
-my %perlvar=%{$perlvarref};
+%perlvar=%{$perlvarref};
 undef $perlvarref; # remove since sensitive and not needed
 delete $perlvar{'lonReceipt'}; # remove since sensitive and not needed
 delete $perlvar{'lonSqlAccess'}; # remove since sensitive and not needed
@@ -195,25 +304,48 @@ delete $perlvar{'lonSqlAccess'}; # remov
        }
     }
 }
+# ------------------------------------------------------------ Read domain file
+{
+    my $fh=IO::File->new($perlvar{'lonTabDir'}.'/domain.tab');
+    %domaindescription = ();
+    %domain_auth_def = ();
+    %domain_auth_arg_def = ();
+    if ($fh) {
+       while (<$fh>) {
+           next if (/^(\#|\s*$)/);
+           chomp;
+           my ($domain, $domain_description, $def_auth, $def_auth_arg)
+               = split(/:/,$_,4);
+           $domain_auth_def{$domain}=$def_auth;
+           $domain_auth_arg_def{$domain}=$def_auth_arg;
+           $domaindescription{$domain}=$domain_description;
+       }
+    }
+}
+
 
-print "<h1>Cluster Status ".localtime()."</h1>";
+print "<img src='/adm/lonIcons/lonlogos.gif' align='right' /><h1>LON-CAPA Cluster Status ".localtime()."</h1>";
 print "<form name='prgstat'>\n".
 "<input type='text' name='progress' value='Starting ...' size='100' /><br />".
 "</form>\n";;
 print "<form name='status' method='post'>\n";
+print 'Choose next report: '.&select_form($mode,'mode',%modes).'<hr />';
 &hidden('refresh',$refresh);
 
 # ==================================================== Main Loop over all Hosts
 
 foreach $local (sort keys %hostname) {
+    $host{$local.'_unresponsive_doomed'}=0;
 # -- Check general status
     &statuslist($local,'General');
     my %loncron=&replyhash($local,'/lon-status/loncron_simple.txt',1200);
     if (defined($loncron{'local_error'})) {
 	$host{$local.'_loncron'}='Could not determine.';
+	$host{$local.'_unresponsive_doomed'}++;
     } else {
 	if ((time-$loncron{'time'})>$oneday) {
 	    $host{$local.'_loncron'}='Stale.';
+	    $host{$local.'_unresponsive_doomed'}++;
 	} else {
 	}
     }
@@ -222,13 +354,20 @@ foreach $local (sort keys %hostname) {
     my %userstatus=&replyhash($local,'/cgi-bin/userstatus.pl?simple',600);
     if (defined($userstatus{'local_error'})) {
 	$host{$local.'_userstatus'}='Could not determine.';
+	$host{$local.'_unresponsive_doomed'}++;
     } else {
+	$host{$local.'_users_doomed'}=$userstatus{'Active'};
+	$host{$local.'_users'}=$userstatus{'Active'};
+	my ($sload,$mload,$lload)=split(/ /,$userstatus{'loadavg'});
+	$host{$local.'_load_doomed'}=$mload;
+	$host{$local.'_load'}=$userstatus{'loadavg'};
     }
 # -- Check mysql status
     &statuslist($local,'Database');
-    my %mysql=&replyhash($local,'/lon-status/mysql.txt',1200);
+    my %mysql=&replyhash($local,'/lon-status/mysql.txt',3600);
     if (defined($mysql{'local_error'})) {
 	$host{$local.'_mysql'}='Could not determine.';
+	$host{$local.'_unresponsive_doomed'}++;
     } else {
 	if ((time-$mysql{'time'})>(7*$oneday)) {
 	    if ($hostrole{$local} eq 'library') {
@@ -246,13 +385,15 @@ foreach $local (sort keys %hostname) {
     }
 # -- Check rpm status
     &statuslist($local,'RPMs');
-    my %checkrpms=&replyhash($local,'/lon-status/checkrpms.txt',2400);
+    my %checkrpms=&replyhash($local,'/lon-status/checkrpms.txt',7200);
     if (defined($checkrpms{'local_error'})) {
 	$host{$local.'_checkrpms'}='Could not determine.';
+	$host{$local.'_unresponsive_doomed'}++;
     } else {
 	if ((time-$checkrpms{'time'})>(4*$oneday)) {
 	    $host{$local.'_checkrpms'}='Stale.';
 	    $host{$local.'_checkrpms_doomed'}=50;
+	    $host{$local.'_unresponsive_doomed'}++;
 	} elsif ($checkrpms{'status'} eq 'fail') {
 	    $host{$local.'_checkrpms'}='Could not checked RPMs.';
 	    $host{$local.'_checkrpms_doomed'}=100;
@@ -274,11 +415,48 @@ foreach $local (sort keys %hostname) {
 	    $host{$local.'_notconnected_doomed'}++;
 	}
     }
-# Eventually, use doomed count
-    &serverstatus($local);
-}
-
 # =============================================================== End Mail Loop
+}
+&statuslist('Done.');
+# ====================================================================== Output
+    if ($mode=~/\_doomed$/) {
+# Output by doomedness
+	foreach (&doomedness($mode)) {
+	    &serverstatus($_);
+	}
+    } elsif ($mode eq 'connections') {
+	print 
+       "<table cellspacing='3' cellpadding='3' border='0' bgcolor='#225522'>".
+       "<tr><td bgcolor='#225522'>&nbsp;</td>";
+	foreach (sort keys %hostname) {
+	    my $remote=$_;
+	    print '<th bgcolor="#DDDDBB">'.$remote.'</th>';
+	}
+	print "</tr>\n";
+# connection matrix
+	foreach (sort keys %hostname) {
+	    my $local=$_;
+	    print '<tr><th bgcolor="#DDDDBB">'.$local.'</th>';
+	    foreach (sort keys %hostname) {
+		my $remote=$_;
+		if ($connectionstatus{$local.'_TO_'.$remote} eq 'not_yet') {
+		    print '<td bgcolor="#FFFFFF">&nbsp;</td>';
+		} elsif ($connectionstatus{$local.'_TO_'.$remote} eq 'ok') {
+		    print 
+'<td bgcolor="#BBDDBB"><font color="#225522" face="arial"><b>ok</b></td>';
+		} else {
+		    print 
+		  '<td bgcolor="#DDBBBB"><font color="#552222" size="-2">'.
+		  $connectionstatus{$local.'_TO_'.$remote}.'<br />';
+		    &lonc($local); &lond($remote);
+		    print '</td>';
+		}
+	    }
+	    print "</tr>\n";
+	}
+    }
+    print "</table>";
+# ============================================================== Close, refresh
 print "</form><script>";
 $runtime=time-$starttime;
 if ($runtime>=$refresh) {