--- loncom/cgi/clusterstatus.pl	2003/07/31 15:35:02	1.9
+++ loncom/cgi/clusterstatus.pl	2003/08/20 01:34:25	1.19
@@ -3,10 +3,11 @@ $|=1;
 # The LearningOnline Network with CAPA
 # Cluster Status
 #
-# $Id: clusterstatus.pl,v 1.9 2003/07/31 15:35:02 www Exp $
+# $Id: clusterstatus.pl,v 1.19 2003/08/20 01:34:25 www Exp $
 
 use lib '/home/httpd/lib/perl/';
 use LONCAPA::Configuration;
+use strict;
 
 use LWP::UserAgent();
 use HTTP::Headers;
@@ -19,6 +20,32 @@ my %connectionstatus=();
 my %perlvar=();
 
 my $mode;
+my $concount=0;
+my $fromcache;
+
+my %domaindescription = ();
+my %domain_auth_def = ();
+my %domain_auth_arg_def = ();
+my %domain_lang_def=();
+my %domain_city=();
+my %domain_longi=();
+my %domain_lati=();
+
+my %hostname=();
+my %hostip=();
+my %hostdom=();
+my %hostrole=();
+my %libserv=();
+
+my $maxusers=0;
+my $maxload=0;
+my $totalusers=0;
+
+my %FORM=();
+
+my $stat_total=0;
+my $stat_notyet=0;
+my $stat_fromcache=0;
 
 sub select_form {
     my ($def,$name,%hash) = @_;
@@ -47,13 +74,26 @@ sub hidden {
 
 sub request {
     my ($local,$url,$cachetime)=@_;
+    $cachetime*=(0.5+rand);
     my $key=&key($local,$url);
     my $reply='';
+    $stat_total++;
+# if fromcache flag is set, only return cached values
+    if ($fromcache) {
+	if ($FORM{$key.'_time'}) {
+	    return $FORM{$key};
+	    $stat_fromcache++;
+	} else {
+	    return 'not_yet';
+	    $stat_notyet++;
+	}
+    }
+# normal mode, refresh when expired or not yet present
     if ($FORM{$key.'_time'}) {
 	if ((time-$FORM{$key.'_time'})<$cachetime) {
 	    $reply=$FORM{$key};
 	    &hidden($key.'_time',$FORM{$key.'_time'});
-	    &hidden($key.'_fromcache',1);
+	    $stat_fromcache++;
 	}
     }
     unless ($reply) {
@@ -61,7 +101,7 @@ sub request {
 	    $reply='local_unknown'; 
 	} else {
 
-	    my $ua=new LWP::UserAgent(timeout => 20);
+	    my $ua=new LWP::UserAgent(timeout => 10);
     
 	    my $request=new HTTP::Request('GET',
 					  "http://".$hostname{$local}.$url);
@@ -91,10 +131,17 @@ sub connected {
     unless ($hostname{$remote}) { return 'remote_unknown'; }
     my $url='/cgi-bin/ping.pl?'.$remote;
 #
-# Slowly phase this in: if not cached, only do 10 percent of the cases 
+# Slowly phase this in: if not cached, only do 5 percent of the cases,
+# but always do the first five. 
 #
     unless ($FORM{&key($local,$url)}) {
-	unless (rand>0.9) { return 'not_yet'; }
+	unless (($concount<=5) || (rand>0.95)) {
+	    $stat_total++;
+	    $stat_notyet++; 
+	    return 'not_yet'; 
+	} else {
+	    $concount++;
+	}
     }
 #
 # Actually do the query
@@ -169,20 +216,47 @@ sub server {
     print &otherwindow($local,'/server-status','Server Status');
 }
 
+sub announcement {
+    my $local=shift;
+    print &otherwindow($local,'/announcement.txt','Announcement');
+}
+
+# ========================================================= Produce a green bar
+sub bar {
+    my $parm=shift;
+    my $number=int($parm+0.5);
+    print "<table><tr><td bgcolor='#225522'><font color='#225522'>";
+    for (my $i=0;$i<$number;$i++) {
+	print "+";
+    }
+    print "</font></table>";
+}
+
 # ========================================================== Show server status
 
 sub serverstatus {
-    my $local=shift;
+    my ($local,$trouble)=@_;
     print (<<ENDHEADER);
+<a name="$local" />
 <table width="100%" bgcolor="#225522" cellspacing="2" cellpadding="2" border="0">
 <tr><td bgcolor="#BBDDBB"><font color="#225522" face="arial"><b>
 $local $hostdom{$local}</b> <tt>($hostname{$local}; $hostrole{$local})</tt>
 <br />$domaindescription{$hostdom{$local}}
-</font></th></tr><tr><td bgcolor="DDDDBB"><font color="#225522">
+$domain_city{$hostdom{$local}}
+</font></th></tr><tr><td bgcolor="#DDDDBB"><font color="#225522">
 ENDHEADER
     &login($local);&server($local);&users($local);&versions($local);
+    &announcement($local);
     &loncron($local);&lond($local);&lonc($local);&runloncron($local);
-    print "</font></td></tr><tr><td bgcolor='#BBBBBB'>";
+    print "</font></td></tr>";
+    if ($trouble) {
+	print ("<tr><td bgcolor='#DDBBBB'><font color='#552222' size='+2'>$trouble</font></td></tr>");
+    }
+    print "<tr><td bgcolor='#BBBBBB'>";
+# version
+    if ($host{$local.'_version'}) {
+	print "<br />Version: ".$host{$local.'_version'}
+    }
 # load
     if (($host{$local.'_load_doomed'}>0.5) || ($mode eq 'load_doomed')) {
 	print "<br />Load: ".$host{$local.'_load'}
@@ -200,6 +274,19 @@ ENDHEADER
     if ($host{$local.'_mysql'}) {
 	print "<br />MySQL Database: ".$host{$local.'_mysql'}
     }
+# connections
+    if ($host{$local.'_notconnected'}) {
+	print "<br />Not connected: ";
+	foreach (split(/ /,$host{$local.'_notconnected'})) {
+	    if ($_) {
+		print " <a href='#$_'>$_</a>";
+	    }
+	}
+    }
+# errors
+    if ($host{$local.'_errors'}) {
+	print "<br />loncron errors: ".$host{$local.'_errors'};
+    }
     print "</td></tr></table><br />";
 }
 
@@ -220,6 +307,260 @@ sub doomedness {
     return sort { $alldoomed{$b} <=> $alldoomed{$a} } @allhosts;
 }
 
+sub resetvars {
+   $maxusers=0;
+   $maxload=0;
+   $totalusers=0;
+   $stat_total=0;
+   $stat_notyet=0;
+   $stat_fromcache=0;
+   $concount=0;
+   undef %host;
+   %host=();
+}
+
+sub mainloop {
+    &resetvars();
+# ==================================================== Main Loop over all Hosts
+
+foreach my $local (sort keys %hostname) {
+    $host{$local.'_unresponsive_doomed'}=0;
+# -- Check general status
+    &statuslist($local,'General');
+    my %loncron=&replyhash($local,'/lon-status/loncron_simple.txt',1200);
+    if (defined($loncron{'local_error'})) {
+	$host{$local.'_loncron'}='Could not determine.';
+	$host{$local.'_unresponsive_doomed'}++;
+    } else {
+	if ((time-$loncron{'time'})>$oneday) {
+	    $host{$local.'_loncron'}='Stale.';
+	    $host{$local.'_unresponsive_doomed'}++;
+	} else {
+	    $host{$local.'_loncron_doomed'}=$loncron{'notices'}
+	                                 +4*$loncron{'warnings'}
+	                               +100*$loncron{'errors'};
+	    $host{$local.'_errors'}=$loncron{'errors'};
+	}
+    }
+# -- Check version
+    &statuslist($local,'Version');
+    my $version=&request($local,'/lon-status/version.txt',7200);
+    if ($version eq 'local_error') {
+	$host{$local.'_version'}='Could not determine.';
+	$host{$local.'_unresponsive_doomed'}++;
+    } else {
+	$host{$local.'_version'}=$version;
+    }
+# -- Check user status
+    &statuslist($local,'Users');
+    my %userstatus=&replyhash($local,'/cgi-bin/userstatus.pl?simple',600);
+    if (defined($userstatus{'local_error'})) {
+	$host{$local.'_userstatus'}='Could not determine.';
+	$host{$local.'_unresponsive_doomed'}++;
+    } else {
+	$host{$local.'_users_doomed'}=$userstatus{'Active'};
+	$host{$local.'_users'}=$userstatus{'Active'};
+	unless ($host{$local.'_users'}) { $host{$local.'_users'}=0; }
+	if ($host{$local.'_users'}>$maxusers) { 
+	    $maxusers=$host{$local.'_users'};
+	}
+	$totalusers+=$host{$local.'_users'};
+	my ($sload,$mload,$lload)=split(/ /,$userstatus{'loadavg'});
+	$host{$local.'_load_doomed'}=$mload;
+	if ($mload>$maxload) { 
+	    $maxload=$mload;
+	}
+	$host{$local.'_load'}=$userstatus{'loadavg'};
+    }
+# -- Check mysql status
+    &statuslist($local,'Database');
+    my %mysql=&replyhash($local,'/lon-status/mysql.txt',3600);
+    if (defined($mysql{'local_error'})) {
+	$host{$local.'_mysql'}='Could not determine.';
+	$host{$local.'_unresponsive_doomed'}++;
+    } else {
+	if ((time-$mysql{'time'})>(7*$oneday)) {
+	    if ($hostrole{$local} eq 'library') {
+		$host{$local.'_mysql'}='Stale.';
+		$host{$local.'_mysql_doomed'}=1;
+	    }
+	    if ($mysql{'mysql'} eq 'defunct') {
+		$host{$local.'_mysql'}='Defunct (maybe stale).';
+		$host{$local.'_mysql_doomed'}=2;
+	    }
+	} elsif ($mysql{'mysql'} eq 'defunct') {
+	    $host{$local.'_mysql'}='Defunct.';
+	    $host{$local.'_mysql_doomed'}=3;
+	}
+    }
+# -- Check rpm status
+    &statuslist($local,'RPMs');
+    my %checkrpms=&replyhash($local,'/lon-status/checkrpms.txt',7200);
+    if (defined($checkrpms{'local_error'})) {
+	$host{$local.'_checkrpms'}='Could not determine.';
+	$host{$local.'_unresponsive_doomed'}++;
+    } else {
+	if ((time-$checkrpms{'time'})>(4*$oneday)) {
+	    $host{$local.'_checkrpms'}='Stale.';
+	    $host{$local.'_checkrpms_doomed'}=50;
+	    $host{$local.'_unresponsive_doomed'}++;
+	} elsif ($checkrpms{'status'} eq 'fail') {
+	    $host{$local.'_checkrpms'}='Could not checked RPMs.';
+	    $host{$local.'_checkrpms_doomed'}=100;
+	} elsif ($checkrpms{'rpmcount'}) {
+	    $host{$local.'_checkrpms'}='Outdated RPMs: '.
+		$checkrpms{'rpmcount'};
+	    $host{$local.'_checkrpms_doomed'}=$checkrpms{'rpmcount'};
+	}
+    }
+# -- Check connections
+    &statuslist($local,'Connections');
+    $host{$local.'_notconnected'}='';
+    $host{$local.'_notconnected_doomed'}=0;
+    foreach my $remote (sort keys %hostname) {
+	my $status=&connected($local,$remote);
+	$connectionstatus{$local.'_TO_'.$remote}=$status;
+	unless (($status eq 'ok') || ($status eq 'not_yet')) {
+	    $host{$local.'_notconnected'}.=' '.$remote;
+	    $host{$local.'_notconnected_doomed'}++;
+	}
+    }
+# =============================================================== End Main Loop
+}
+
+}
+
+sub reports {
+# ====================================================================== Output
+    if ($mode=~/\_doomed$/) {
+# Output by doomedness
+	foreach (&doomedness($mode)) {
+	    &serverstatus($_);
+	}
+    } elsif ($mode eq 'connections') {
+	print 
+       "<table cellspacing='3' cellpadding='3' border='0' bgcolor='#225522'>".
+       "<tr><td bgcolor='#225522'>&nbsp;</td>";
+	foreach my $remote (sort keys %hostname) {
+	    print '<td bgcolor="#DDDDBB">'.$remote.'</td>';
+	}
+	print "</tr>\n";
+# connection matrix
+	foreach my $local (sort keys %hostname) {
+	    print '<tr><td bgcolor="#DDDDBB">'.$local.'</td>';
+	    foreach my $remote (sort keys %hostname) {
+		if ($connectionstatus{$local.'_TO_'.$remote} eq 'not_yet') {
+		    my $cellcolor='#FFFFFF';
+		    if ($local eq $remote) { $cellcolor='#DDDDDD'; }
+		    print '<td bgcolor="'.$cellcolor.'"><font color="#555522" size="-2">not yet tested</font></td>';
+		} elsif ($connectionstatus{$local.'_TO_'.$remote} eq 'ok') {
+		    my $cellcolor='#BBDDBB';
+		    if ($local eq $remote) { $cellcolor='#99DD99'; }
+		    print 
+'<td bgcolor="'.$cellcolor.'"><font color="#225522" face="arial"><b>ok</b></td>';
+		} else {
+		    my $cellcolor='#DDBBBB';
+		    if ($connectionstatus{$local.'_TO_'.$remote} eq 'local_error') {
+			if ($local eq $remote) { 
+			    $cellcolor='#DD88AA'; 
+			} else {
+			    $cellcolor='#DDAACC';
+			}
+		    } else {
+			if ($local eq $remote) { $cellcolor='#DD9999'; }
+		    }
+		    print 
+		  '<td bgcolor="'.$cellcolor.'"><font color="#552222" size="-2">'.
+		  $connectionstatus{$local.'_TO_'.$remote}.'<br />';
+		    &lonc($local); &lond($remote);
+		    print '</td>';
+		}
+	    }
+	    print "</tr>\n";
+	}
+	print "</table>";
+    } elsif ($mode eq 'users') {
+# Users
+	if ($maxusers) {
+	    my $factor=50/$maxusers;
+	    print "<h3>Total active user(s): $totalusers</h3>". 
+       "<table cellspacing='3' cellpadding='3' border='0' bgcolor='#225522'>";
+
+	    foreach my $local (sort keys %hostname) {
+		if (defined($host{$local.'_users'})) {
+		    print 
+'<tr><td bgcolor="#BBDDBB"><font face="arial" color="#225522" size="+1">'.$local.
+			'</font><br /><font size="-2">'.
+			$domaindescription{$hostdom{$local}}.
+		       '</font></td><td bgcolor="#DDDDBB">';
+		    &users($local);
+		    print 
+	      '</td><td bgcolor="#DDDDBB"><font face="arial" color="#225522">'.
+	      $host{$local.'_users'}.'</font></td><td bgcolor="#DDDDBB"';
+		    &bar($factor*$host{$local.'_users'});
+		    print "</td></tr>\n";
+		}
+	    }
+	    print "</table>";
+	} else {
+	    print "No active users logged in.";
+	}
+    } elsif ($mode eq 'load') {
+# Load
+	if ($maxload) {
+	    my $factor=50/$maxload; 
+	    print
+       "<table cellspacing='3' cellpadding='3' border='0' bgcolor='#225522'>";
+	    foreach my $local (sort keys %hostname) {
+		if (defined($host{$local.'_load_doomed'})) {
+		    print 
+'<tr><td bgcolor="#BBDDBB"><font face="arial" color="#225522" size="+1">'.
+                        $local.
+			'</font><br /><font size="-2">'.
+			$domaindescription{$hostdom{$local}}.
+		       '</font></td><td bgcolor="#DDDDBB">';
+		    &server($local);
+		    print 
+	      '</td><td bgcolor="#DDDDBB"><font face="arial" color="#225522">'.
+	      $host{$local.'_load_doomed'}.'</font></td><td bgcolor="#DDDDBB"';
+		    &bar($factor*$host{$local.'_load_doomed'});
+		    print "</td></tr>\n";
+		}
+	    }
+	    print "</table>";
+	} else {
+	    print "No workload.";
+	}
+    } elsif ($mode eq 'trouble') {
+	my $count=0;
+	foreach my $local (sort keys %hostname) {
+	    my $trouble='';
+	    if ($host{$local.'_unresponsive_doomed'}>3) {
+		$trouble='Does not respond to several queries.<br />';
+	    }
+	    if ($host{$local.'_errors'}) {
+		$trouble='Has loncron errors.<br />';
+	    } elsif ($host{$local.'_loncron_doomed'}>600) {
+		$trouble='High loncron count.<br />';
+	    }
+	    if ($host{$local.'_load_doomed'}>5) {
+		$trouble='High load.<br />';
+	    }
+	    if ($host{$local.'_users_doomed'}>200) {
+		$trouble='High user volume.<br />';
+	    }
+	    if ($host{$local.'_mysql_doomed'}>1) {
+		$trouble='MySQL database apparently offline.<br />';
+	    }
+	    if ($host{$local.'_checkrpms_doomed'}>100) {
+		$trouble='RPMs outdated.<br />';
+	    }
+	    if ($trouble) { $count++; &serverstatus($local,$trouble); }
+	}
+	unless ($count) { print "No mayor trouble."; }
+    }
+}
+
 # ====================================================================== Status
 sub statuslist {
     my ($local,$what)=@_;
@@ -227,7 +568,8 @@ sub statuslist {
 "<script>document.prgstat.progress.value='Testing $local ($hostname{$local}): $what';</script>\n";
 }
 
-#
+# =============================================================================
+# =============================================================================
 # Main program
 #
 # ========================================================= Get form parameters
@@ -256,7 +598,7 @@ foreach $pair (@pairs) {
 
 # ====================================================== Determine refresh rate
 
-my $refresh=(($FORM{'refresh'}=~/^\d+$/)?$FORM{'refresh'}:60);
+my $refresh=(($FORM{'refresh'}=~/^\d+$/)?$FORM{'refresh'}:30);
 if ($refresh<30) { $refresh=30; }
 my $starttime=time;
 
@@ -271,13 +613,14 @@ my %modes=('trouble' => 'Trouble',
 	   'load_doomed' => 'Doomed: Load',
 	   'unresponsive_doomed' => 'Doomed: Status could not be determined',
 	   'users' => 'User Report',
+	   'load' => 'Load Report',
 	   'connections' => 'Connections Matrix');
 
 $mode=$FORM{'mode'};
 unless ($modes{$mode}) { $mode='trouble'; }
 # ================================================================ Send Headers
 print "Content-type: text/html\n\n".
-    "<html><body bgcolor=#FFFFFF>\n";
+    "<html><body bgcolor='#FFFFFF'>\n";
 # -------------------- Read loncapa.conf (and by default, loncapa_apache.conf).
 my $perlvarref=LONCAPA::Configuration::read_conf('loncapa.conf');
 %perlvar=%{$perlvarref};
@@ -289,7 +632,6 @@ delete $perlvar{'lonSqlAccess'}; # remov
 {
     my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab");
 
-    $total=0;
     while (my $configline=<$config>) {
        $configline=~s/#.*$//;
        unless ($configline=~/\w/) { next; } 
@@ -298,7 +640,6 @@ delete $perlvar{'lonSqlAccess'}; # remov
        $hostdom{$id}=$domain;
        $hostrole{$id}=$role;
        $hostip{$id}=$ip;
-       $total++;
        if (($role eq 'library') && ($id ne $perlvar{'lonHostID'})) {
 	   $libserv{$id}=$name;
        }
@@ -307,24 +648,24 @@ delete $perlvar{'lonSqlAccess'}; # remov
 # ------------------------------------------------------------ Read domain file
 {
     my $fh=IO::File->new($perlvar{'lonTabDir'}.'/domain.tab');
-    %domaindescription = ();
-    %domain_auth_def = ();
-    %domain_auth_arg_def = ();
     if ($fh) {
        while (<$fh>) {
            next if (/^(\#|\s*$)/);
            chomp;
-           my ($domain, $domain_description, $def_auth, $def_auth_arg)
-               = split(/:/,$_,4);
-           $domain_auth_def{$domain}=$def_auth;
+           my ($domain, $domain_description, $def_auth, $def_auth_arg,
+	       $def_lang, $city, $longi, $lati) = split(/:/,$_);
+	   $domain_auth_def{$domain}=$def_auth;
            $domain_auth_arg_def{$domain}=$def_auth_arg;
-           $domaindescription{$domain}=$domain_description;
+	   $domaindescription{$domain}=$domain_description;
+	   $domain_lang_def{$domain}=$def_lang;
+	   $domain_city{$domain}=$city;
+	   $domain_longi{$domain}=$longi;
+	   $domain_lati{$domain}=$lati;
        }
     }
 }
 
-
-print "<h1>Cluster Status ".localtime()."</h1>";
+print "<img src='/adm/lonIcons/lonlogos.gif' align='right' /><h1>LON-CAPA Cluster Status ".localtime()."</h1>";
 print "<form name='prgstat'>\n".
 "<input type='text' name='progress' value='Starting ...' size='100' /><br />".
 "</form>\n";;
@@ -332,107 +673,37 @@ print "<form name='status' method='post'
 print 'Choose next report: '.&select_form($mode,'mode',%modes).'<hr />';
 &hidden('refresh',$refresh);
 
-# ==================================================== Main Loop over all Hosts
-
-foreach $local (sort keys %hostname) {
-    $host{$local.'_unresponsive_doomed'}=0;
-# -- Check general status
-    &statuslist($local,'General');
-    my %loncron=&replyhash($local,'/lon-status/loncron_simple.txt',1200);
-    if (defined($loncron{'local_error'})) {
-	$host{$local.'_loncron'}='Could not determine.';
-	$host{$local.'_unresponsive_doomed'}++;
-    } else {
-	if ((time-$loncron{'time'})>$oneday) {
-	    $host{$local.'_loncron'}='Stale.';
-	    $host{$local.'_unresponsive_doomed'}++;
-	} else {
-	}
-    }
-# -- Check user status
-    &statuslist($local,'Users');
-    my %userstatus=&replyhash($local,'/cgi-bin/userstatus.pl?simple',600);
-    if (defined($userstatus{'local_error'})) {
-	$host{$local.'_userstatus'}='Could not determine.';
-	$host{$local.'_unresponsive_doomed'}++;
+    if (!$FORM{'runonetime'}) {
+	print 
+   "<h3>Gathering initial cluster data</h3>This may take some time ...<br />";
+	$fromcache=0;
+	&mainloop();
+	&statuslist('Done initial run.');
+	&reports();
     } else {
-	$host{$local.'_users_doomed'}=$userstatus{'Active'};
-	$host{$local.'_users'}=$userstatus{'Active'};
-	my ($sload,$mload,$lload)=split(/ /,$userstatus{'loadavg'});
-	$host{$local.'_load_doomed'}=$mload;
-	$host{$local.'_load'}=$userstatus{'loadavg'};
-    }
-# -- Check mysql status
-    &statuslist($local,'Database');
-    my %mysql=&replyhash($local,'/lon-status/mysql.txt',3600);
-    if (defined($mysql{'local_error'})) {
-	$host{$local.'_mysql'}='Could not determine.';
-	$host{$local.'_unresponsive_doomed'}++;
-    } else {
-	if ((time-$mysql{'time'})>(7*$oneday)) {
-	    if ($hostrole{$local} eq 'library') {
-		$host{$local.'_mysql'}='Stale.';
-		$host{$local.'_mysql_doomed'}=1;
-	    }
-	    if ($mysql{'mysql'} eq 'defunct') {
-		$host{$local.'_mysql'}='Defunct (maybe stale).';
-		$host{$local.'_mysql_doomed'}=2;
-	    }
-	} elsif ($mysql{'mysql'} eq 'defunct') {
-	    $host{$local.'_mysql'}='Defunct.';
-	    $host{$local.'_mysql_doomed'}=3;
-	}
-    }
-# -- Check rpm status
-    &statuslist($local,'RPMs');
-    my %checkrpms=&replyhash($local,'/lon-status/checkrpms.txt',7200);
-    if (defined($checkrpms{'local_error'})) {
-	$host{$local.'_checkrpms'}='Could not determine.';
-	$host{$local.'_unresponsive_doomed'}++;
-    } else {
-	if ((time-$checkrpms{'time'})>(4*$oneday)) {
-	    $host{$local.'_checkrpms'}='Stale.';
-	    $host{$local.'_checkrpms_doomed'}=50;
-	    $host{$local.'_unresponsive_doomed'}++;
-	} elsif ($checkrpms{'status'} eq 'fail') {
-	    $host{$local.'_checkrpms'}='Could not checked RPMs.';
-	    $host{$local.'_checkrpms_doomed'}=100;
-	} elsif ($checkrpms{'rpmcount'}) {
-	    $host{$local.'_checkrpms'}='Outdated RPMs: '.
-		$checkrpms{'rpmcount'};
-	    $host{$local.'_checkrpms_doomed'}=$checkrpms{'rpmcount'};
-	}
-    }
-# -- Check connections
-    &statuslist($local,'Connections');
-    $host{$local.'_notconnected'}='';
-    $host{$local.'_notconnected_doomed'}=0;
-    foreach $remote (sort keys %hostname) {
-	my $status=&connected($local,$remote);
-	$connectionstatus{$local.'_TO_'.$remote}=$status;
-	unless (($status eq 'ok') || ($status eq 'not_yet')) {
-	    $host{$local.'_notconnected'}.=' '.$remote;
-	    $host{$local.'_notconnected_doomed'}++;
-	}
-    }
-# =============================================================== End Mail Loop
-}
-&statuslist('Done.');
-# ====================================================================== Output
-    if ($mode=~/\_doomed$/) {
-# Output by doomedness
-	foreach (&doomedness($mode)) {
-	    &serverstatus($_);
-	}
-    }
+	$fromcache=1;
+	&mainloop();
+	&statuslist('Done gathering cached data');
+	&reports();
+	$fromcache=0;
+	&mainloop();
+    }
+    &hidden('runonetime',1);
+print '<tt><br />Total number of queries: '.$stat_total.
+          '<br />Percent complete: '.
+	  int(($stat_total-$stat_notyet)/$stat_total*100.).
+	  '<br />Percent from cache: '.
+          int($stat_fromcache/$stat_total*100.).'</tt>';
+
 # ============================================================== Close, refresh
 print "</form><script>";
-$runtime=time-$starttime;
-if ($runtime>=$refresh) {
-    print 'document.status.submit();';
+my $runtime=time-$starttime;
+if (($refresh-$runtime)<0) {
+    print "document.status.submit();";
 } else {
-    $refreshtime=int(1000*($refresh-$runtime));
-    print "setTimeout('document.status.submit()',$refreshtime);";
+    my $refreshtime=int(1000*($refresh-$runtime));
+    print "setTimeout('document.status.submit()',$refreshtime);\n".
+          "document.prgstat.progress.value='Will automatically refresh ($refresh secs refresh cycle)'";
 }
 print "</script></body></html>";
 exit 0;