loncom/cgi/clusterstatus.pl - view

File: [LON-CAPA] / loncom / cgi / clusterstatus.pl
Revision 1.14: download - view: text, annotated - select for diffs
Fri Aug 1 19:20:26 2003 UTC (20 years, 11 months ago) by www
Branches: MAIN
CVS tags: HEAD

* Making timeout even smaller. If 10 secs are not enough, there is some
trouble - which is the thing we want to detect
* Always deal with 5 connections - makes no difference in the beginning,
but makes sure that the final obnoxious holes get filed up towards the
end
* Make diagonal (connections to self) better visible in connections matrix

1: #!/usr/bin/perl 2: $|=1; 3: # The LearningOnline Network with CAPA 4: # Cluster Status 5: # 6: # $Id: clusterstatus.pl,v 1.14 2003/08/01 19:20:26 www Exp $ 7: 8: use lib '/home/httpd/lib/perl/'; 9: use LONCAPA::Configuration; 10: 11: use LWP::UserAgent(); 12: use HTTP::Headers; 13: use IO::File; 14: 15: my %host=(); 16: my $oneday=60*60*24; 17: 18: my %connectionstatus=(); 19: my %perlvar=(); 20: 21: my $mode; 22: my $concount=0; 23: 24: sub select_form { 25: my ($def,$name,%hash) = @_; 26: my $selectform = "<select name=\"$name\" size=\"1\">\n"; 27: foreach (sort keys %hash) { 28: $selectform.="<option value=\"$_\" ". 29: ($_ eq $def ? 'selected' : ''). 30: ">".$hash{$_}."</option>\n"; 31: } 32: $selectform.="</select>"; 33: return $selectform; 34: } 35: 36: 37: sub key { 38: my ($local,$url)=@_; 39: my $key=$local.'_'.$url; 40: $key=~s/\W/\_/gs; 41: return $key; 42: } 43: 44: sub hidden { 45: my ($name,$value)=@_; 46: print "\n<input type='hidden' name='$name' value='$value' />"; 47: } 48: 49: sub request { 50: my ($local,$url,$cachetime)=@_; 51: $cachetime*=(0.5+rand); 52: my $key=&key($local,$url); 53: my $reply=''; 54: if ($FORM{$key.'_time'}) { 55: if ((time-$FORM{$key.'_time'})<$cachetime) { 56: $reply=$FORM{$key}; 57: &hidden($key.'_time',$FORM{$key.'_time'}); 58: &hidden($key.'_fromcache',1); 59: } 60: } 61: unless ($reply) { 62: unless ($hostname{$local}) { 63: $reply='local_unknown'; 64: } else { 65: 66: my $ua=new LWP::UserAgent(timeout => 10); 67: 68: my $request=new HTTP::Request('GET', 69: "http://".$hostname{$local}.$url); 70: $request->authorization_basic('lonadm','litelite'); 71: 72: my $response=$ua->request($request); 73: 74: unless ($response->is_success) { 75: $reply='local_error'; 76: } else { 77: $reply=$response->content; 78: chomp($reply); 79: } 80: } 81: &hidden($key.'_time',time); 82: } 83: &hidden($key,$reply); 84: return $reply; 85: } 86: 87: # ============================================= Are local and remote connected? 88: sub connected { 89: my ($local,$remote)=@_; 90: $local=~s/\W//g; 91: $remote=~s/\W//g; 92: 93: unless ($hostname{$remote}) { return 'remote_unknown'; } 94: my $url='/cgi-bin/ping.pl?'.$remote; 95: # 96: # Slowly phase this in: if not cached, only do 5 percent of the cases, 97: # but always do the first five. 98: # 99: unless ($FORM{&key($local,$url)}) { 100: unless (($concount<=5) || (rand>0.95)) { 101: return 'not_yet'; 102: } else { 103: $concount++; 104: } 105: } 106: # 107: # Actually do the query 108: # 109: &statuslist($local,'connecting '.$remote); 110: my $reply=&request($local,$url,3600); 111: $reply=(split("\n",$reply))[0]; 112: $reply=~s/\W//g; 113: if ($reply ne $remote) { return $reply; } 114: return 'ok'; 115: } 116: # ============================================================ Get a reply hash 117: 118: sub replyhash { 119: my %returnhash=(); 120: foreach (split(/\&/,&request(@_))) { 121: my ($name,$value)=split(/\=/,$_); 122: if ($name) { 123: unless ($value) { $value=''; } 124: $returnhash{$name}=$value; 125: } 126: } 127: return %returnhash; 128: } 129: 130: # ================================================================ Link to host 131: 132: sub otherwindow { 133: my ($local,$url,$label)=@_; 134: return 135: " <a href='http://$hostname{$local}$url' target='newwin$local'>$label</a> "; 136: } 137: 138: sub login { 139: my $local=shift; 140: print &otherwindow($local,'/adm/login?domain='.$perlvar{'lonDefDomain'}, 141: 'Login'); 142: } 143: 144: sub runloncron { 145: my $local=shift; 146: print &otherwindow($local,'/cgi-bin/loncron.pl','Run loncron'); 147: } 148: 149: sub loncron { 150: my $local=shift; 151: print &otherwindow($local,'/lon-status','loncron'); 152: } 153: 154: sub lonc { 155: my $local=shift; 156: print &otherwindow($local,'/lon-status/loncstatus.txt','lonc'); 157: } 158: 159: sub lond { 160: my $local=shift; 161: print &otherwindow($local,'/lon-status/londstatus.txt','lond'); 162: } 163: 164: sub users { 165: my $local=shift; 166: print &otherwindow($local,'/cgi-bin/userstatus.pl','Users'); 167: } 168: 169: sub versions { 170: my $local=shift; 171: print &otherwindow($local,'/cgi-bin/lonversions.pl','Versions'); 172: } 173: 174: sub server { 175: my $local=shift; 176: print &otherwindow($local,'/server-status','Server Status'); 177: } 178: 179: # ========================================================= Produce a green bar 180: sub bar { 181: my $parm=shift; 182: my $number=int($parm+0.5); 183: print "<table><tr><td bgcolor='#225522'><font color='#225522'>"; 184: for (my $i=0;$i<$number;$i++) { 185: print "+"; 186: } 187: print "</font></table>"; 188: } 189: 190: # ========================================================== Show server status 191: 192: sub serverstatus { 193: my ($local,$trouble)=@_; 194: print (<<ENDHEADER); 195: <a name="$local" /> 196: <table width="100%" bgcolor="#225522" cellspacing="2" cellpadding="2" border="0"> 197: <tr><td bgcolor="#BBDDBB"><font color="#225522" face="arial"><b> 198: $local $hostdom{$local}</b> <tt>($hostname{$local}; $hostrole{$local})</tt> 199: <br />$domaindescription{$hostdom{$local}} 200: </font></th></tr><tr><td bgcolor="#DDDDBB"><font color="#225522"> 201: ENDHEADER 202: &login($local);&server($local);&users($local);&versions($local); 203: &loncron($local);&lond($local);&lonc($local);&runloncron($local); 204: print "</font></td></tr>"; 205: if ($trouble) { 206: print ("<tr><td bgcolor='#DDBBBB'><font color='#552222' size='+2'>$trouble</font></td></tr>"); 207: } 208: print "<tr><td bgcolor='#BBBBBB'>"; 209: # load 210: if (($host{$local.'_load_doomed'}>0.5) || ($mode eq 'load_doomed')) { 211: print "<br />Load: ".$host{$local.'_load'} 212: } 213: # users 214: if (($host{$local.'_users_doomed'}>10) || ($mode eq 'users_doomed')) { 215: print "<br />Active Users: ".$host{$local.'_users'} 216: } 217: 218: # checkrpms 219: if ($host{$local.'_checkrpms'}) { 220: print "<br />RPMs: ".$host{$local.'_checkrpms'} 221: } 222: # mysql 223: if ($host{$local.'_mysql'}) { 224: print "<br />MySQL Database: ".$host{$local.'_mysql'} 225: } 226: # connections 227: if ($host{$local.'_notconnected'}) { 228: print "<br />Not connected: "; 229: foreach (split(/ /,$host{$local.'_notconnected'})) { 230: if ($_) { 231: print " <a href='#$_'>$_</a>"; 232: } 233: } 234: } 235: # errors 236: if ($host{$local.'_errors'}) { 237: print "<br />loncron errors: ".$host{$local.'_errors'}; 238: } 239: print "</td></tr></table><br />"; 240: } 241: 242: # =========================================================== Doomedness sorted 243: 244: sub doomedness { 245: my $crit=shift; 246: my %alldoomed=(); 247: my @allhosts=(); 248: foreach (keys %host) { 249: if ($_=~/^(\w+)\_$crit$/) { 250: if ($host{$_}) { 251: push (@allhosts,$1); 252: $alldoomed{$1}=$host{$_}; 253: } 254: } 255: } 256: return sort { $alldoomed{$b} <=> $alldoomed{$a} } @allhosts; 257: } 258: 259: # ====================================================================== Status 260: sub statuslist { 261: my ($local,$what)=@_; 262: print 263: "<script>document.prgstat.progress.value='Testing $local ($hostname{$local}): $what';</script>\n"; 264: } 265: 266: # 267: # Main program 268: # 269: # ========================================================= Get form parameters 270: my $buffer; 271: 272: read(STDIN, $buffer, $ENV{'CONTENT_LENGTH'}); 273: my @pairs=split(/&/,$buffer); 274: my $pair; my $name; my $value; 275: undef %FORM; 276: %FORM=(); 277: foreach $pair (@pairs) { 278: ($name,$value) = split(/=/,$pair); 279: $value =~ tr/+/ /; 280: $value =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C",hex($1))/eg; 281: $FORM{$name}=$value; 282: } 283: 284: $buffer=$ENV{'QUERY_STRING'}; 285: @pairs=split(/&/,$buffer); 286: foreach $pair (@pairs) { 287: ($name,$value) = split(/=/,$pair); 288: $value =~ tr/+/ /; 289: $value =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C",hex($1))/eg; 290: $FORM{$name}=$value; 291: } 292: 293: # ====================================================== Determine refresh rate 294: 295: my $refresh=(($FORM{'refresh'}=~/^\d+$/)?$FORM{'refresh'}:120); 296: if ($refresh<30) { $refresh=30; } 297: my $starttime=time; 298: 299: # ============================================================== Determine mode 300: 301: my %modes=('trouble' => 'Trouble', 302: 'users_doomed' => 'Doomed: Users', 303: 'loncron_doomed' => 'Doomed: General (loncron)', 304: 'mysql_doomed' => 'Doomed: Database (mysql)', 305: 'notconnected_doomed' => 'Doomed: Connections', 306: 'checkrpms_doomed' => 'Doomed: RPMs', 307: 'load_doomed' => 'Doomed: Load', 308: 'unresponsive_doomed' => 'Doomed: Status could not be determined', 309: 'users' => 'User Report', 310: 'load' => 'Load Report', 311: 'connections' => 'Connections Matrix'); 312: 313: $mode=$FORM{'mode'}; 314: unless ($modes{$mode}) { $mode='trouble'; } 315: # ================================================================ Send Headers 316: print "Content-type: text/html\n\n". 317: "<html><body bgcolor='#FFFFFF'>\n"; 318: # -------------------- Read loncapa.conf (and by default, loncapa_apache.conf). 319: my $perlvarref=LONCAPA::Configuration::read_conf('loncapa.conf'); 320: %perlvar=%{$perlvarref}; 321: undef $perlvarref; # remove since sensitive and not needed 322: delete $perlvar{'lonReceipt'}; # remove since sensitive and not needed 323: delete $perlvar{'lonSqlAccess'}; # remove since sensitive and not needed 324: 325: # ------------------------------------------------------------- Read hosts file 326: { 327: my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab"); 328: 329: $total=0; 330: while (my $configline=<$config>) { 331: $configline=~s/#.*$//; 332: unless ($configline=~/\w/) { next; } 333: my ($id,$domain,$role,$name,$ip)=split(/:/,$configline); 334: $hostname{$id}=$name; 335: $hostdom{$id}=$domain; 336: $hostrole{$id}=$role; 337: $hostip{$id}=$ip; 338: $total++; 339: if (($role eq 'library') && ($id ne $perlvar{'lonHostID'})) { 340: $libserv{$id}=$name; 341: } 342: } 343: } 344: # ------------------------------------------------------------ Read domain file 345: { 346: my $fh=IO::File->new($perlvar{'lonTabDir'}.'/domain.tab'); 347: %domaindescription = (); 348: %domain_auth_def = (); 349: %domain_auth_arg_def = (); 350: if ($fh) { 351: while (<$fh>) { 352: next if (/^(\#|\s*$)/); 353: chomp; 354: my ($domain, $domain_description, $def_auth, $def_auth_arg) 355: = split(/:/,$_,4); 356: $domain_auth_def{$domain}=$def_auth; 357: $domain_auth_arg_def{$domain}=$def_auth_arg; 358: $domaindescription{$domain}=$domain_description; 359: } 360: } 361: } 362: 363: print "<img src='/adm/lonIcons/lonlogos.gif' align='right' /><h1>LON-CAPA Cluster Status ".localtime()."</h1>"; 364: print "<form name='prgstat'>\n". 365: "<input type='text' name='progress' value='Starting ...' size='100' /><br />". 366: "</form>\n";; 367: print "<form name='status' method='post'>\n"; 368: print 'Choose next report: '.&select_form($mode,'mode',%modes).'<hr />'; 369: &hidden('refresh',$refresh); 370: 371: # ==================================================== Main Loop over all Hosts 372: 373: my $maxusers=0; 374: my $maxload=0; 375: my $totalusers=0; 376: 377: foreach $local (sort keys %hostname) { 378: $host{$local.'_unresponsive_doomed'}=0; 379: # -- Check general status 380: &statuslist($local,'General'); 381: my %loncron=&replyhash($local,'/lon-status/loncron_simple.txt',1200); 382: if (defined($loncron{'local_error'})) { 383: $host{$local.'_loncron'}='Could not determine.'; 384: $host{$local.'_unresponsive_doomed'}++; 385: } else { 386: if ((time-$loncron{'time'})>$oneday) { 387: $host{$local.'_loncron'}='Stale.'; 388: $host{$local.'_unresponsive_doomed'}++; 389: } else { 390: $host{$local.'_loncron_doomed'}=$loncron{'notices'} 391: +4*$loncron{'warnings'} 392: +100*$loncron{'errors'}; 393: $host{$local.'_errors'}=$loncron{'errors'}; 394: } 395: } 396: # -- Check user status 397: &statuslist($local,'Users'); 398: my %userstatus=&replyhash($local,'/cgi-bin/userstatus.pl?simple',600); 399: if (defined($userstatus{'local_error'})) { 400: $host{$local.'_userstatus'}='Could not determine.'; 401: $host{$local.'_unresponsive_doomed'}++; 402: } else { 403: $host{$local.'_users_doomed'}=$userstatus{'Active'}; 404: $host{$local.'_users'}=$userstatus{'Active'}; 405: unless ($host{$local.'_users'}) { $host{$local.'_users'}=0; } 406: if ($host{$local.'_users'}>$maxusers) { 407: $maxusers=$host{$local.'_users'}; 408: } 409: $totalusers+=$host{$local.'_users'}; 410: my ($sload,$mload,$lload)=split(/ /,$userstatus{'loadavg'}); 411: $host{$local.'_load_doomed'}=$mload; 412: if ($mload>$maxload) { 413: $maxload=$mload; 414: } 415: $host{$local.'_load'}=$userstatus{'loadavg'}; 416: } 417: # -- Check mysql status 418: &statuslist($local,'Database'); 419: my %mysql=&replyhash($local,'/lon-status/mysql.txt',3600); 420: if (defined($mysql{'local_error'})) { 421: $host{$local.'_mysql'}='Could not determine.'; 422: $host{$local.'_unresponsive_doomed'}++; 423: } else { 424: if ((time-$mysql{'time'})>(7*$oneday)) { 425: if ($hostrole{$local} eq 'library') { 426: $host{$local.'_mysql'}='Stale.'; 427: $host{$local.'_mysql_doomed'}=1; 428: } 429: if ($mysql{'mysql'} eq 'defunct') { 430: $host{$local.'_mysql'}='Defunct (maybe stale).'; 431: $host{$local.'_mysql_doomed'}=2; 432: } 433: } elsif ($mysql{'mysql'} eq 'defunct') { 434: $host{$local.'_mysql'}='Defunct.'; 435: $host{$local.'_mysql_doomed'}=3; 436: } 437: } 438: # -- Check rpm status 439: &statuslist($local,'RPMs'); 440: my %checkrpms=&replyhash($local,'/lon-status/checkrpms.txt',7200); 441: if (defined($checkrpms{'local_error'})) { 442: $host{$local.'_checkrpms'}='Could not determine.'; 443: $host{$local.'_unresponsive_doomed'}++; 444: } else { 445: if ((time-$checkrpms{'time'})>(4*$oneday)) { 446: $host{$local.'_checkrpms'}='Stale.'; 447: $host{$local.'_checkrpms_doomed'}=50; 448: $host{$local.'_unresponsive_doomed'}++; 449: } elsif ($checkrpms{'status'} eq 'fail') { 450: $host{$local.'_checkrpms'}='Could not checked RPMs.'; 451: $host{$local.'_checkrpms_doomed'}=100; 452: } elsif ($checkrpms{'rpmcount'}) { 453: $host{$local.'_checkrpms'}='Outdated RPMs: '. 454: $checkrpms{'rpmcount'}; 455: $host{$local.'_checkrpms_doomed'}=$checkrpms{'rpmcount'}; 456: } 457: } 458: # -- Check connections 459: &statuslist($local,'Connections'); 460: $host{$local.'_notconnected'}=''; 461: $host{$local.'_notconnected_doomed'}=0; 462: foreach $remote (sort keys %hostname) { 463: my $status=&connected($local,$remote); 464: $connectionstatus{$local.'_TO_'.$remote}=$status; 465: unless (($status eq 'ok') || ($status eq 'not_yet')) { 466: $host{$local.'_notconnected'}.=' '.$remote; 467: $host{$local.'_notconnected_doomed'}++; 468: } 469: } 470: # =============================================================== End Mail Loop 471: } 472: &statuslist('Done.'); 473: # ====================================================================== Output 474: if ($mode=~/\_doomed$/) { 475: # Output by doomedness 476: foreach (&doomedness($mode)) { 477: &serverstatus($_); 478: } 479: } elsif ($mode eq 'connections') { 480: print 481: "<table cellspacing='3' cellpadding='3' border='0' bgcolor='#225522'>". 482: "<tr><td bgcolor='#225522'> </td>"; 483: foreach my $remote (sort keys %hostname) { 484: print '<th bgcolor="#DDDDBB">'.$remote.'</th>'; 485: } 486: print "</tr>\n"; 487: # connection matrix 488: foreach my $local (sort keys %hostname) { 489: print '<tr><th bgcolor="#DDDDBB">'.$local.'</th>'; 490: foreach my $remote (sort keys %hostname) { 491: if ($connectionstatus{$local.'_TO_'.$remote} eq 'not_yet') { 492: my $cellcolor='#FFFFFF'; 493: if ($local eq $remote) { $cellcolor='#DDDDDD'; } 494: print '<td bgcolor="'.$cellcolor.'"><font color="#555522" size="-2">not yet tested</font></td>'; 495: } elsif ($connectionstatus{$local.'_TO_'.$remote} eq 'ok') { 496: my $cellcolor='#BBDDBB'; 497: if ($local eq $remote) { $cellcolor='#99DD99'; } 498: print 499: '<td bgcolor="'.$cellcolor.'"><font color="#225522" face="arial"><b>ok</b></td>'; 500: } else { 501: my $cellcolor='#DDBBBB'; 502: if ($connectionstatus{$local.'_TO_'.$remote} eq 'local_error') { 503: if ($local eq $remote) { 504: $cellcolor='#DD88AA'; 505: } else { 506: $cellcolor='#DDAACC'; 507: } 508: } else { 509: if ($local eq $remote) { $cellcolor='#DD9999'; } 510: } 511: print 512: '<td bgcolor="'.$cellcolor.'"><font color="#552222" size="-2">'. 513: $connectionstatus{$local.'_TO_'.$remote}.'<br />'; 514: &lonc($local); &lond($remote); 515: print '</td>'; 516: } 517: } 518: print "</tr>\n"; 519: } 520: print "</table>"; 521: } elsif ($mode eq 'users') { 522: # Users 523: if ($maxusers) { 524: my $factor=50/$maxusers; 525: print "<h3>Total active user(s): $totalusers</h3>". 526: "<table cellspacing='3' cellpadding='3' border='0' bgcolor='#225522'>"; 527: 528: foreach $local (sort keys %hostname) { 529: if (defined($host{$local.'_users'})) { 530: print 531: '<tr><th bgcolor="#BBDDBB"><font face="arial" color="#225522" size="+1">'.$local. 532: '</font></th><td bgcolor="#DDDDBB">'; 533: &users($local); 534: print 535: '</td><td bgcolor="#DDDDBB"><font face="arial" color="#225522">'. 536: $host{$local.'_users'}.'</font></td><td bgcolor="#DDDDBB"'; 537: &bar($factor*$host{$local.'_users'}); 538: print "</td></tr>\n"; 539: } 540: } 541: print "</table>"; 542: } else { 543: print "No active users logged in."; 544: } 545: } elsif ($mode eq 'load') { 546: # Load 547: if ($maxload) { 548: my $factor=50/$maxload; 549: print 550: "<table cellspacing='3' cellpadding='3' border='0' bgcolor='#225522'>"; 551: foreach $local (sort keys %hostname) { 552: if (defined($host{$local.'_load_doomed'})) { 553: print 554: '<tr><th bgcolor="#BBDDBB"><font face="arial" color="#225522" size="+1">'. 555: $local. 556: '</font></th><td bgcolor="#DDDDBB">'; 557: &server($local); 558: print 559: '</td><td bgcolor="#DDDDBB"><font face="arial" color="#225522">'. 560: $host{$local.'_load_doomed'}.'</font></td><td bgcolor="#DDDDBB"'; 561: &bar($factor*$host{$local.'_load_doomed'}); 562: print "</td></tr>\n"; 563: } 564: } 565: print "</table>"; 566: } else { 567: print "No workload."; 568: } 569: } elsif ($mode eq 'trouble') { 570: my $count=0; 571: foreach $local (sort keys %hostname) { 572: my $trouble=''; 573: if ($host{$local.'_errors'}) { 574: $trouble='Has loncron errors.<br />'; 575: } elsif ($host{$local.'_loncron_doomed'}>600) { 576: $trouble='High loncron count.<br />'; 577: } 578: if ($host{$local.'_load_doomed'}>5) { 579: $trouble='High load.<br />'; 580: } 581: if ($host{$local.'_users_doomed'}>200) { 582: $trouble='High user volume.<br />'; 583: } 584: if ($host{$local.'_mysql_doomed'}>1) { 585: $trouble='MySQL database apparently offline.<br />'; 586: } 587: if ($host{$local.'_checkrpms_doomed'}>100) { 588: $trouble='RPMs outdated.<br />'; 589: } 590: if ($trouble) { $count++; &serverstatus($local,$trouble); } 591: } 592: unless ($count) { print "No mayor trouble."; } 593: } 594: # ============================================================== Close, refresh 595: print "</form><script>"; 596: $runtime=time-$starttime; 597: if (($refresh-$runtime)<30) { 598: print "setTimeout('document.status.submit()',30000);\n". 599: "document.prgstat.progress.value='Will automatically refresh.'"; 600: } else { 601: $refreshtime=int(1000*($refresh-$runtime)); 602: print "setTimeout('document.status.submit()',$refreshtime);\n". 603: "document.prgstat.progress.value='Will automatically refresh ($refresh secs refresh cycle)'"; 604: } 605: print "</script></body></html>"; 606: exit 0;