--- loncom/loncron	2000/02/08 17:34:24	1.3
+++ loncom/loncron	2000/12/23 16:56:46	1.11
@@ -9,6 +9,8 @@
 #
 # 7/14,7/15,7/19,7/21,7/22,11/18,
 # 2/8 Gerd Kortemeyer
+# Dec 00 Scott Harrison
+# 12/23 Gerd Kortemeyer
 
 use IO::File;
 use IO::Socket;
@@ -110,7 +112,10 @@ print $fh (<<ENDHEADERS);
 <ol>
 <li><a href="#configuration">Configuration</a>
 <li><a href="#machine">Machine Information</a>
+<li><a href="#tmp">Temporary Files</a>
+<li><a href="#tokens">Session Tokens</a>
 <li><a href="#httpd">httpd</a>
+<li><a href="#lonsql">lonsql</a>
 <li><a href="#lond">lond</a>
 <li><a href="#lonc">lonc</a>
 <li><a href="#lonnet">lonnet</a>
@@ -153,7 +158,7 @@ close (LOADAVGH);
 print $fh "<tt>$loadavg</tt>";
 
 @parts=split(/\s+/,$loadavg);
-if ($parts[1]>3.0) {
+if ($parts[1]>4.0) {
     $errors++;
 } elsif ($parts[1]>2.0) {
     $warnings++;
@@ -171,17 +176,59 @@ while ($line=<DFH>) {
    $usage=$parts[4];
    $usage=~s/\W//g;
    if ($usage>90) { 
-      $errors++; 
+      $warnings++; 
    } elsif ($usage>80) {
       $warnings++;
    } elsif ($usage>60) {
       $notices++;
    }
-   if ($usage>95) { $errors++; }
+   if ($usage>95) { $warnings++; $warnings++ }
 }
 close (DFH);
 print $fh "</pre>";
 &errout($fh);
+
+# --------------------------------------------------------------- clean out tmp
+print $fh '<hr><a name="tmp"><h2>Temporary Files</h2>';
+$cleaned=0;
+while ($fname=<$perlvar{'lonDaemons'}/tmp/*>) {
+                          my ($dev,$ino,$mode,$nlink,
+                              $uid,$gid,$rdev,$size,
+                              $atime,$mtime,$ctime,
+                              $blksize,$blocks)=stat($fname);
+                          $now=time;
+                          $since=$now-$mtime;
+                          if ($since>$perlvar{'lonExpire'}) {
+                              $cleaned++;
+                              unlink("$fname");
+                          }
+    
+}
+print $fh "Cleaned up ".$cleaned." files.";
+
+# ------------------------------------------------------------ clean out lonIDs
+print $fh '<hr><a name="tokens"><h2>Session Tokens</h2>';
+$cleaned=0;
+$active=0;
+while ($fname=<$perlvar{'lonIDsDir'}/*>) {
+                          my ($dev,$ino,$mode,$nlink,
+                              $uid,$gid,$rdev,$size,
+                              $atime,$mtime,$ctime,
+                              $blksize,$blocks)=stat($fname);
+                          $now=time;
+                          $since=$now-$mtime;
+                          if ($since>$perlvar{'lonExpire'}) {
+                              $cleaned++;
+                              print $fh "Unlinking $fname<br>";
+                              unlink("$fname");
+                          } else {
+                              $active++;
+                          }
+    
+}
+print $fh "<p>Cleaned up ".$cleaned." stale session token(s).";
+print $fh "<h3>$active open session(s)</h3>";
+
 # ----------------------------------------------------------------------- httpd
 
 print $fh '<hr><a name="httpd"><h2>httpd</h2><h3>Access Log</h3><pre>';
@@ -200,6 +247,93 @@ while ($line=<DFH>) {
 close (DFH);
 print $fh "</pre>";
 &errout($fh);
+
+
+# ---------------------------------------------------------------------- lonsql
+#
+# Do not run for now
+#
+if ($perlvar{'lonRole'} eq "library" && 1==0) {
+
+    print $fh '<hr><a name="lonsql"><h2>lonsql</h2><h3>Log</h3><pre>';
+    
+    if (-e "$perlvar{'lonDaemons'}/logs/lonsql.log"){
+	open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonsql.log|");
+	while ($line=<DFH>) { 
+	    print $fh "$line";
+	    if ($line=~/INFO/) { $notices++; }
+	    if ($line=~/WARNING/) { $notices++; }
+	    if ($line=~/CRITICAL/) { $warnings++; }
+	};
+	close (DFH);
+    }
+    print $fh "</pre>";
+    
+    my $lonsqlfile="$perlvar{'lonDaemons'}/logs/lonsql.pid";
+    
+    if (-e $lonsqlfile) {
+	my $lfh=IO::File->new("$lonsqlfile");
+	my $lonsqlpid=<$lfh>;
+	chomp($lonsqlpid);
+	if (kill 0 => $lonsqlpid) {
+	    print $fh "<h3>lonsql at pid $lonsqlpid responding</h3>";
+	} else {
+	    $errors++; $errors++;
+	    print $fh "<h3>lonsql at pid $lonsqlpid not responding</h3>";
+	}
+    } else {
+	$errors++;
+	print $fh "<h3>lonsql not running, trying to start</h3>";
+	system("$perlvar{'lonDaemons'}/lonsql");
+	sleep 120;
+	if (-e $lonsqlfile) {
+	    print $fh "Seems like it started ...<p>";
+	    my $lfh=IO::File->new("$lonsqlfile");
+	    my $lonsqlpid=<$lfh>;
+	    chomp($lonsqlpid);
+	    sleep 30;
+	    if (kill 0 => $lonsqlpid) {
+		print $fh "<h3>lonsql at pid $lonsqlpid responding</h3>";
+	    } else {
+		$errors++; $errors++;
+		print $fh "<h3>lonsql at pid $lonsqlpid not responding</h3>";
+		print $fh "Give it one more try ...<p>";
+		system("$perlvar{'lonDaemons'}/lonsql");
+		sleep 120;
+	    }
+	} else {
+	    print $fh "Seems like that did not work!<p>";
+	    $errors++;
+	}
+	if (-e "$perlvar{'lonDaemons'}/logs/lonsql.log"){
+	    print $fh "<p><pre>";
+	    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonsql.log|");
+	    while ($line=<DFH>) { 
+		print $fh "$line";
+		if ($line=~/WARNING/) { $notices++; }
+		if ($line=~/CRITICAL/) { $notices++; }
+	    };
+	    close (DFH);
+	    print $fh "</pre>";
+	}
+    }
+
+    $fname="$perlvar{'lonDaemons'}/logs/lonsql.log";
+
+    my ($dev,$ino,$mode,$nlink,
+	$uid,$gid,$rdev,$size,
+	$atime,$mtime,$ctime,
+	$blksize,$blocks)=stat($fname);
+
+    if ($size>40000) {
+	print $fh "Rotating logs ...<p>";
+	rename("$fname.2","$fname.3");
+	rename("$fname.1","$fname.2");
+	rename("$fname","$fname.1");
+    }
+
+    &errout($fh);
+}
 # ------------------------------------------------------------------------ lond
 
 print $fh '<hr><a name="lond"><h2>lond</h2><h3>Log</h3><pre>';
@@ -209,8 +343,8 @@ open (DFH,"tail -n100 $perlvar{'lonDaemo
 while ($line=<DFH>) { 
    print $fh "$line";
    if ($line=~/INFO/) { $notices++; }
-   if ($line=~/WARNING/) { $warnings++; }
-   if ($line=~/CRITICAL/) { $errors++; }
+   if ($line=~/WARNING/) { $notices++; }
+   if ($line=~/CRITICAL/) { $warnings++; }
 };
 close (DFH);
 }
@@ -218,17 +352,37 @@ print $fh "</pre>";
 
 my $londfile="$perlvar{'lonDaemons'}/logs/lond.pid";
 
-if (-e $londfile) {
+my $restartflag=1;
+if (-e $londfile) {    
    my $lfh=IO::File->new("$londfile");
    my $londpid=<$lfh>;
    chomp($londpid);
    if (kill 0 => $londpid) {
       print $fh "<h3>lond at pid $londpid responding</h3>";
+      $restartflag=0;
    } else {
-      $errors++; $errors++;
+      $errors++;
       print $fh "<h3>lond at pid $londpid not responding</h3>";
+      # Intelligently handle this.
+      # Possibility #1: there is no process
+      # Solution: remove .pid file and restart
+      if (getpgrp($londpid)==-1) {
+	  unlink($londfile);
+	  $restartflag=1;
+      }
+      else {
+      # Possibility #2: there is a live process that is not responding
+      #                 for an unknown reason
+      # Solution: kill parent and children processes, remove .pid and restart
+	  `killall -9 lond`;
+	  unlink($londfile);
+	  $restartflag=1;
+      }
+      print $fh 
+	  "<h3>Deciding to clean up stale .pid file and restart lond</h3>";
    }
-} else {
+} 
+if ($restartflag==1) {
    $errors++;
    print $fh "<h3>lond not running, trying to start</h3>";
    system("$perlvar{'lonDaemons'}/lond");
@@ -257,9 +411,8 @@ if (-e $londfile) {
     open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lond.log|");
     while ($line=<DFH>) { 
       print $fh "$line";
-      if ($line=~/INFO/) { $notices++; }
-      if ($line=~/WARNING/) { $warnings++; }
-      if ($line=~/CRITICAL/) { $errors++; }
+      if ($line=~/WARNING/) { $notices++; }
+      if ($line=~/CRITICAL/) { $notices++; }
     };
     close (DFH);
     print $fh "</pre>";
@@ -290,8 +443,8 @@ open (DFH,"tail -n100 $perlvar{'lonDaemo
 while ($line=<DFH>) { 
    print $fh "$line";
    if ($line=~/INFO/) { $notices++; }
-   if ($line=~/WARNING/) { $warnings++; }
-   if ($line=~/CRITICAL/) { $errors++; }
+   if ($line=~/WARNING/) { $notices++; }
+   if ($line=~/CRITICAL/) { $warnings++; }
 };
 close (DFH);
 }
@@ -299,6 +452,7 @@ print $fh "</pre>";
 
 my $loncfile="$perlvar{'lonDaemons'}/logs/lonc.pid";
 
+$restartflag=1;
 if (-e $loncfile) {
    my $lfh=IO::File->new("$loncfile");
    my $loncpid=<$lfh>;
@@ -306,11 +460,30 @@ if (-e $loncfile) {
    if (kill 0 => $loncpid) {
       print $fh "<h3>lonc at pid $loncpid responding, sending USR1</h3>";
       kill USR1 => $loncpid;
+      $restartflag=0;
    } else {
-      $errors++; $errors++;
+      $errors++;
       print $fh "<h3>lonc at pid $loncpid not responding</h3>";
+      # Intelligently handle this.
+      # Possibility #1: there is no process
+      # Solution: remove .pid file and restart
+      if (getpgrp($loncpid)==-1) {
+	  unlink($loncfile);
+	  $restartflag=1;
+      }
+      else {
+      # Possibility #2: there is a live process that is not responding
+      #                 for an unknown reason
+      # Solution: kill parent and children processes, remove .pid and restart
+	  `killall -9 lonc`;
+	  unlink($loncfile);
+	  $restartflag=1;
+      }
+      print $fh 
+	  "<h3>Deciding to clean up stale .pid file and restart lonc</h3>";
    }
-} else {
+} 
+if ($restartflag==1) {
    $errors++;
    print $fh "<h3>lonc not running, trying to start</h3>";
    system("$perlvar{'lonDaemons'}/lonc");
@@ -339,9 +512,8 @@ if (-e $loncfile) {
     open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonc.log|");
     while ($line=<DFH>) { 
       print $fh "$line";
-      if ($line=~/INFO/) { $notices++; }
-      if ($line=~/WARNING/) { $warnings++; }
-      if ($line=~/CRITICAL/) { $errors++; }
+      if ($line=~/WARNING/) { $notices++; }
+      if ($line=~/CRITICAL/) { $notices++; }
     };
     close (DFH);
     print $fh "</pre>";
@@ -374,7 +546,7 @@ while ($line=<DFH>) {
 };
 close (DFH);
 }
-print $fh "</pre><h3>Perm Log</h3>";
+print $fh "</pre><h3>Perm Log</h3><pre>";
 
 if (-e "$perlvar{'lonDaemons'}/logs/lonnet.perm.log") {
     open(DFH,"tail -n10 $perlvar{'lonDaemons'}/logs/lonnet.perm.log|");