--- loncom/loncron	2000/02/08 17:34:24	1.3
+++ loncom/loncron	2000/12/08 23:33:36	1.9
@@ -153,7 +153,7 @@ close (LOADAVGH);
 print $fh "<tt>$loadavg</tt>";
 
 @parts=split(/\s+/,$loadavg);
-if ($parts[1]>3.0) {
+if ($parts[1]>4.0) {
     $errors++;
 } elsif ($parts[1]>2.0) {
     $warnings++;
@@ -171,13 +171,13 @@ while ($line=<DFH>) {
    $usage=$parts[4];
    $usage=~s/\W//g;
    if ($usage>90) { 
-      $errors++; 
+      $warnings++; 
    } elsif ($usage>80) {
       $warnings++;
    } elsif ($usage>60) {
       $notices++;
    }
-   if ($usage>95) { $errors++; }
+   if ($usage>95) { $warnings++; $warnings++ }
 }
 close (DFH);
 print $fh "</pre>";
@@ -200,6 +200,91 @@ while ($line=<DFH>) {
 close (DFH);
 print $fh "</pre>";
 &errout($fh);
+
+
+# ------------------------------------------------------------------------ lonsql
+
+if ($perlvar{'lonRole'} eq "library" && 1==0) {
+
+    print $fh '<hr><a name="lond"><h2>lonsql</h2><h3>Log</h3><pre>';
+    
+    if (-e "$perlvar{'lonDaemons'}/logs/lonsql.log"){
+	open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonsql.log|");
+	while ($line=<DFH>) { 
+	    print $fh "$line";
+	    if ($line=~/INFO/) { $notices++; }
+	    if ($line=~/WARNING/) { $notices++; }
+	    if ($line=~/CRITICAL/) { $warnings++; }
+	};
+	close (DFH);
+    }
+    print $fh "</pre>";
+    
+    my $lonsqlfile="$perlvar{'lonDaemons'}/logs/lonsql.pid";
+    
+    if (-e $lonsqlfile) {
+	my $lfh=IO::File->new("$lonsqlfile");
+	my $lonsqlpid=<$lfh>;
+	chomp($lonsqlpid);
+	if (kill 0 => $lonsqlpid) {
+	    print $fh "<h3>lonsql at pid $lonsqlpid responding</h3>";
+	} else {
+	    $errors++; $errors++;
+	    print $fh "<h3>lonsql at pid $lonsqlpid not responding</h3>";
+	}
+    } else {
+	$errors++;
+	print $fh "<h3>lonsql not running, trying to start</h3>";
+	system("$perlvar{'lonDaemons'}/lonsql");
+	sleep 120;
+	if (-e $lonsqlfile) {
+	    print $fh "Seems like it started ...<p>";
+	    my $lfh=IO::File->new("$lonsqlfile");
+	    my $lonsqlpid=<$lfh>;
+	    chomp($lonsqlpid);
+	    sleep 30;
+	    if (kill 0 => $lonsqlpid) {
+		print $fh "<h3>lonsql at pid $lonsqlpid responding</h3>";
+	    } else {
+		$errors++; $errors++;
+		print $fh "<h3>lonsql at pid $lonsqlpid not responding</h3>";
+		print $fh "Give it one more try ...<p>";
+		system("$perlvar{'lonDaemons'}/lonsql");
+		sleep 120;
+	    }
+	} else {
+	    print $fh "Seems like that did not work!<p>";
+	    $errors++;
+	}
+	if (-e "$perlvar{'lonDaemons'}/logs/lonsql.log"){
+	    print $fh "<p><pre>";
+	    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonsql.log|");
+	    while ($line=<DFH>) { 
+		print $fh "$line";
+		if ($line=~/WARNING/) { $notices++; }
+		if ($line=~/CRITICAL/) { $notices++; }
+	    };
+	    close (DFH);
+	    print $fh "</pre>";
+	}
+    }
+
+    $fname="$perlvar{'lonDaemons'}/logs/lonsql.log";
+
+    my ($dev,$ino,$mode,$nlink,
+	$uid,$gid,$rdev,$size,
+	$atime,$mtime,$ctime,
+	$blksize,$blocks)=stat($fname);
+
+    if ($size>40000) {
+	print $fh "Rotating logs ...<p>";
+	rename("$fname.2","$fname.3");
+	rename("$fname.1","$fname.2");
+	rename("$fname","$fname.1");
+    }
+
+    &errout($fh);
+}
 # ------------------------------------------------------------------------ lond
 
 print $fh '<hr><a name="lond"><h2>lond</h2><h3>Log</h3><pre>';
@@ -209,8 +294,8 @@ open (DFH,"tail -n100 $perlvar{'lonDaemo
 while ($line=<DFH>) { 
    print $fh "$line";
    if ($line=~/INFO/) { $notices++; }
-   if ($line=~/WARNING/) { $warnings++; }
-   if ($line=~/CRITICAL/) { $errors++; }
+   if ($line=~/WARNING/) { $notices++; }
+   if ($line=~/CRITICAL/) { $warnings++; }
 };
 close (DFH);
 }
@@ -218,17 +303,38 @@ print $fh "</pre>";
 
 my $londfile="$perlvar{'lonDaemons'}/logs/lond.pid";
 
-if (-e $londfile) {
+my $restartflag=1;
+if (-e $londfile) {    
    my $lfh=IO::File->new("$londfile");
    my $londpid=<$lfh>;
    chomp($londpid);
    if (kill 0 => $londpid) {
       print $fh "<h3>lond at pid $londpid responding</h3>";
+      $restartflag=0;
    } else {
-      $errors++; $errors++;
+      $errors++;
       print $fh "<h3>lond at pid $londpid not responding</h3>";
+      # Intelligently handle this.
+      # Possibility #1: there is no process
+      # Solution: remove .pid file and restart
+      if (getpgrp($londpid)==-1) {
+	  unlink($londfile);
+	  $restartflag=1;
+      }
+      else {
+      # Possibility #2: there is a zombie process
+      # Possibility #3: there is a live process that is not responding
+      #                 for an unknown reason
+      # Solution: kill process, remove .pid and restart
+	  `killall -9 lond`;
+	  unlink($londfile);
+	  $restartflag=1;
+      }
+      print $fh 
+	  "<h3>Deciding to clean up stale .pid file and restart lond</h3>";
    }
-} else {
+} 
+if ($restartflag==1) {
    $errors++;
    print $fh "<h3>lond not running, trying to start</h3>";
    system("$perlvar{'lonDaemons'}/lond");
@@ -257,9 +363,8 @@ if (-e $londfile) {
     open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lond.log|");
     while ($line=<DFH>) { 
       print $fh "$line";
-      if ($line=~/INFO/) { $notices++; }
-      if ($line=~/WARNING/) { $warnings++; }
-      if ($line=~/CRITICAL/) { $errors++; }
+      if ($line=~/WARNING/) { $notices++; }
+      if ($line=~/CRITICAL/) { $notices++; }
     };
     close (DFH);
     print $fh "</pre>";
@@ -290,8 +395,8 @@ open (DFH,"tail -n100 $perlvar{'lonDaemo
 while ($line=<DFH>) { 
    print $fh "$line";
    if ($line=~/INFO/) { $notices++; }
-   if ($line=~/WARNING/) { $warnings++; }
-   if ($line=~/CRITICAL/) { $errors++; }
+   if ($line=~/WARNING/) { $notices++; }
+   if ($line=~/CRITICAL/) { $warnings++; }
 };
 close (DFH);
 }
@@ -299,6 +404,7 @@ print $fh "</pre>";
 
 my $loncfile="$perlvar{'lonDaemons'}/logs/lonc.pid";
 
+$restartflag=1;
 if (-e $loncfile) {
    my $lfh=IO::File->new("$loncfile");
    my $loncpid=<$lfh>;
@@ -306,11 +412,31 @@ if (-e $loncfile) {
    if (kill 0 => $loncpid) {
       print $fh "<h3>lonc at pid $loncpid responding, sending USR1</h3>";
       kill USR1 => $loncpid;
+      $restartflag=0;
    } else {
-      $errors++; $errors++;
+      $errors++;
       print $fh "<h3>lonc at pid $loncpid not responding</h3>";
+      # Intelligently handle this.
+      # Possibility #1: there is no process
+      # Solution: remove .pid file and restart
+      if (getpgrp($loncpid)==-1) {
+	  unlink($loncfile);
+	  $restartflag=1;
+      }
+      else {
+      # Possibility #2: there is a zombie process
+      # Possibility #3: there is a live process that is not responding
+      #                 for an unknown reason
+      # Solution: kill process, remove .pid and restart
+	  `killall -9 lonc`;
+	  unlink($loncfile);
+	  $restartflag=1;
+      }
+      print $fh 
+	  "<h3>Deciding to clean up stale .pid file and restart lonc</h3>";
    }
-} else {
+} 
+if ($restartflag==1) {
    $errors++;
    print $fh "<h3>lonc not running, trying to start</h3>";
    system("$perlvar{'lonDaemons'}/lonc");
@@ -339,9 +465,8 @@ if (-e $loncfile) {
     open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonc.log|");
     while ($line=<DFH>) { 
       print $fh "$line";
-      if ($line=~/INFO/) { $notices++; }
-      if ($line=~/WARNING/) { $warnings++; }
-      if ($line=~/CRITICAL/) { $errors++; }
+      if ($line=~/WARNING/) { $notices++; }
+      if ($line=~/CRITICAL/) { $notices++; }
     };
     close (DFH);
     print $fh "</pre>";