--- loncom/loncnew	2003/12/16 16:12:19	1.37
+++ loncom/loncnew	2004/09/20 18:40:06	1.55
@@ -2,7 +2,7 @@
 # The LearningOnline Network with CAPA
 # lonc maintains the connections to remote computers
 #
-# $Id: loncnew,v 1.37 2003/12/16 16:12:19 albertel Exp $
+# $Id: loncnew,v 1.55 2004/09/20 18:40:06 albertel Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -35,12 +35,12 @@
 #    - Add ability to create/negotiate lond connections (done).
 #    - Add general logic for dispatching requests and timeouts. (done).
 #    - Add support for the lonc/lond requests.          (done).
-#    - Add logging/status monitoring.
-#    - Add Signal handling - HUP restarts. USR1 status report.
+#    - Add logging/status monitoring.                    (done)
+#    - Add Signal handling - HUP restarts. USR1 status report. (done)
 #    - Add Configuration file I/O                       (done).
-#    - Add management/status request interface.
+#    - Add management/status request interface.         (done)
 #    - Add deferred request capability.                  (done)
-#    - Detect transmission timeouts.
+#    - Detect transmission timeouts.                     (done)
 #
 
 use strict;
@@ -63,11 +63,6 @@ use LONCAPA::Configuration;
 use LONCAPA::HashIterator;
 
 
-#
-#   Disable all signals we might receive from outside for now.
-#
-
-
 # Read the httpd configuration file to get perl variables
 # normally set in apache modules:
 
@@ -87,7 +82,9 @@ my $ClientConnection = 0;	# Uniquifier f
 
 my $DebugLevel = 0;
 my $NextDebugLevel= 2;		# So Sigint can toggle this.
-my $IdleTimeout= 3600;		# Wait an hour before pruning connections.
+my $IdleTimeout= 600;		# Wait 10 minutes before pruning connections.
+
+my $LogTransactions = 0;	# When True, all transactions/replies get logged.
 
 #
 #  The variables below are only used by the child processes.
@@ -105,6 +102,9 @@ my $Status          = "";	# Current stat
 my $RecentLogEntry  = "";
 my $ConnectionRetries=2;	# Number of connection retries allowed.
 my $ConnectionRetriesLeft=2;	# Number of connection retries remaining.
+my $LondVersion     = "unknown"; # Version of lond we talk with.
+my $KeyMode         = "";       # e.g. ssl, local, insecure from last connect.
+my $LondConnecting  = 0;       # True when a connection is being built.
 
 #
 #   The hash below gives the HTML format for log messages
@@ -112,10 +112,10 @@ my $ConnectionRetriesLeft=2;	# Number of
 #    
 my %LogFormats;
 
-$LogFormats{"CRITICAL"} = "<font color=red>CRITICAL: %s</font>";
-$LogFormats{"SUCCESS"}  = "<font color=green>SUCCESS: %s</font>";
-$LogFormats{"INFO"}     = "<font color=yellow>INFO: %s</font>";
-$LogFormats{"WARNING"}  = "<font color=blue>WARNING: %s</font>";
+$LogFormats{"CRITICAL"} = "<font color='red'>CRITICAL: %s</font>";
+$LogFormats{"SUCCESS"}  = "<font color='green'>SUCCESS: %s</font>";
+$LogFormats{"INFO"}     = "<font color='yellow'>INFO: %s</font>";
+$LogFormats{"WARNING"}  = "<font color='blue'>WARNING: %s</font>";
 $LogFormats{"DEFAULT"}  = " %s ";
 
 
@@ -158,9 +158,9 @@ host and the time will be formatted into
 =cut
 
 sub Log {
-    my $severity = shift;
-    my $message  = shift;
-   
+
+    my ($severity, $message) = @_;
+
     if(!$LogFormats{$severity}) {
 	$severity = "DEFAULT";
     }
@@ -195,8 +195,10 @@ Returns the name of the host that a sock
 =cut
 
 sub GetPeername {
-    my $connection = shift;
-    my $AdrFamily  = shift;
+
+
+    my ($connection, $AdrFamily) = @_;
+
     my $peer       = $connection->peername();
     my $peerport;
     my $peerip;
@@ -219,18 +221,20 @@ Invoked to issue a debug message.
 =cut
 
 sub Debug {
-    my $level   = shift;
-    my $message = shift;
+
+    my ($level, $message) = @_;
+
     if ($level <= $DebugLevel) {
 	Log("INFO", "-Debug- $message host = $RemoteHost");
     }
 }
 
 sub SocketDump {
-    my $level = shift;
-    my $socket= shift;
+
+    my ($level, $socket) = @_;
+
     if($level <= $DebugLevel) {
-	$socket->Dump();
+	$socket->Dump(-1);	# Ensure it will get dumped.
     }
 }
 
@@ -261,10 +265,19 @@ sub ShowStatus {
 =cut
 sub SocketTimeout {
     my $Socket = shift;
-    
+    Log("WARNING", "A socket timeout was detected");
+    Debug(5, " SocketTimeout called: ");
+    $Socket->Dump(0);
+    if(exists($ActiveTransactions{$Socket})) {
+	FailTransaction($ActiveTransactions{$Socket});
+    }
     KillSocket($Socket);	# A transaction timeout also counts as
                                 # a connection failure:
     $ConnectionRetriesLeft--;
+    if($ConnectionRetriesLeft <= 0) {
+	Log("CRITICAL", "Host marked DEAD: ".GetServerHost());
+    }
+
 }
 #----------------------------- Timer management ------------------------
 
@@ -278,10 +291,14 @@ Invoked  each timer tick.
 
 
 sub Tick {
+    my ($Event)       = @_;
+    my $clock_watcher = $Event->w;
+
     my $client;
     if($ConnectionRetriesLeft > 0) {
 	ShowStatus(GetServerHost()." Connection count: ".$ConnectionCount
-		   ." Retries remaining: ".$ConnectionRetriesLeft);
+		   ." Retries remaining: ".$ConnectionRetriesLeft
+		   ." ($KeyMode)");
     } else {
 	ShowStatus(GetServerHost()." >> DEAD <<");
     }
@@ -294,6 +311,7 @@ sub Tick {
 	if($IdleSeconds > $IdleTimeout) { # Prune a connection...
 	    my $Socket = $IdleConnections->pop();
 	    KillSocket($Socket);
+	    $IdleSeconds = 0;	# Otherwise all connections get trimmed to fast.
 	}
     } else {
 	$IdleSeconds = 0;	# Reset idle count if not idle.
@@ -326,14 +344,20 @@ sub Tick {
 	    if($successCount == 0) { # All connections failed:
 		Debug(5,"Work in queue failed to make any connectiouns\n");
 		EmptyQueue();	# Fail pending transactions with con_lost.
+		CloseAllLondConnections(); # Should all be closed but....
 	    }
 	} else {
 	    ShowStatus(GetServerHost()." >>> DEAD!!! <<<");
 	    Debug(5,"Work in queue, but gave up on connections..flushing\n");
 	    EmptyQueue();	# Connections can't be established.
+	    CloseAllLondConnections(); # Should all already be closed but...
 	}
        
     }
+    if ($ConnectionCount == 0) {
+	$KeyMode = ""; 
+	$clock_watcher->cancel();
+    }
 }
 
 =pod
@@ -374,6 +398,7 @@ long enough, it will be shut down and re
 
 sub ServerToIdle {
     my $Socket   = shift;	# Get the socket.
+    $KeyMode = $Socket->{AuthenticationMode};
     delete($ActiveTransactions{$Socket}); # Server has no transaction
 
     &Debug(5, "Server to idle");
@@ -453,7 +478,7 @@ sub ClientWritable {
 	    } else {		# Partial string sent.
 		$Watcher->data(substr($Data, $result));
 		if($result == 0) {    # client hung up on us!!
-		    Log("INFO", "lonc pipe client hung up on us!");
+		    # Log("INFO", "lonc pipe client hung up on us!");
 		    $Watcher->cancel;
 		    $Socket->shutdown(2);
 		    $Socket->close();
@@ -506,11 +531,14 @@ The transaction that is being completed.
 
 sub CompleteTransaction {
     &Debug(5,"Complete transaction");
-    my $Socket = shift;
-    my $Transaction = shift;
+
+    my ($Socket, $Transaction) = @_;
 
     if (!$Transaction->isDeferred()) { # Normal transaction
 	my $data   = $Socket->GetReply(); # Data to send.
+	if($LogTransactions) {
+	    Log("SUCCESS", "Reply from lond: '$data'");
+	}
 	StartClientReply($Transaction, $data);
     } else {			# Delete deferred transaction file.
 	Log("SUCCESS", "A delayed transaction was completed");
@@ -518,7 +546,9 @@ sub CompleteTransaction {
 	unlink $Transaction->getFile();
     }
 }
+
 =pod
+
 =head1 StartClientReply
 
    Initiates a reply to a client where the reply data is a parameter.
@@ -534,10 +564,10 @@ sub CompleteTransaction {
     The data to send to apached client.
 
 =cut
+
 sub StartClientReply {
-    my $Transaction   = shift;
-    my $data     = shift;
 
+    my ($Transaction, $data) = @_;
 
     my $Client   = $Transaction->getClient();
 
@@ -551,7 +581,9 @@ sub StartClientReply {
 	      cb       => \&ClientWritable,
 	      data     => $data);
 }
+
 =pod
+
 =head2 FailTransaction
 
   Finishes a transaction with failure because the associated lond socket
@@ -561,8 +593,7 @@ sub StartClientReply {
   - The transaction is 'live' in which case we initiate the sending
     of "con_lost" to the client.
 
-Deleting the transaction means killing it from the 
-%ActiveTransactions hash.
+Deleting the transaction means killing it from the %ActiveTransactions hash.
 
 Parameters:
 
@@ -570,20 +601,24 @@ Parameters:
  
    The LondTransaction we are failing.
  
+
 =cut
 
 sub FailTransaction {
     my $transaction = shift;
-    Log("WARNING", "Failing transaction ".$transaction->getRequest());
+    
+    #  If the socket is dead, that's already logged.
+
+    if ($ConnectionRetriesLeft > 0) {
+	Log("WARNING", "Failing transaction "
+	    .$transaction->getRequest());
+    }
     Debug(1, "Failing transaction: ".$transaction->getRequest());
     if (!$transaction->isDeferred()) { # If the transaction is deferred we'll get to it.
 	my $client  = $transaction->getClient();
 	Debug(1," Replying con_lost to ".$transaction->getRequest());
 	StartClientReply($transaction, "con_lost\n");
     }
-    if($ConnectionRetriesLeft <= 0) {
-	Log("CRITICAL", "Host marked dead: ".GetServerHost());
-    }
 
 }
 
@@ -611,7 +646,10 @@ Close all connections open on lond prior
 =cut
 sub CloseAllLondConnections {
     foreach my $Socket (keys %ActiveConnections) {
-	KillSocket($Socket);
+      if(exists($ActiveTransactions{$Socket})) {
+	FailTransaction($ActiveTransactions{$Socket});
+      }
+      KillSocket($Socket);
     }
 }
 =cut
@@ -663,6 +701,7 @@ sub KillSocket {
     #
     if($ConnectionCount == 0) {
 	EmptyQueue();
+	CloseAllLondConnections; # Should all already be closed but...
     }
 }
 
@@ -688,6 +727,17 @@ The connection must echo the challenge b
 The challenge has been replied to.  The we are receiveing the 
 'ok' from the partner.
 
+=head3  State=ReadingVersionString
+
+We have requested the lond version and are reading the
+version back.  Upon completion, we'll store the version away
+for future use(?).
+
+=head3 State=HostSet
+
+We have selected the domain name of our peer (multhomed hosts)
+and are getting the reply (presumably ok) back.
+
 =head3 State=RequestingKey
 
 The ok has been received and we need to send the request for
@@ -742,7 +792,7 @@ sub LondReadable {
     &Debug(2, "Socket->Readable returned: $status");
 
     if($status != 0) {
-	 # bad return from socket read. Currently this means that
+	# bad return from socket read. Currently this means that
 	# The socket has become disconnected. We fail the transaction.
 
 	Log("WARNING",
@@ -760,19 +810,34 @@ sub LondReadable {
     $State = $Socket->GetState(); # Update in case of transition.
     &Debug(6, "After read, state is ".$State);
 
-   if($State eq "Initialized") {
+    if($State eq "Initialized") {
 
 
     } elsif ($State eq "ChallengeReceived") {
 	#  The challenge must be echoed back;  The state machine
 	# in the connection takes care of setting that up.  Just
 	# need to transition to writable:
-
+	
 	$Watcher->cb(\&LondWritable);
 	$Watcher->poll("w");
 
     } elsif ($State eq "ChallengeReplied") {
 
+    } elsif ($State eq "RequestingVersion") {
+	# Need to ask for the version... that is writiability:
+
+	$Watcher->cb(\&LondWritable);
+	$Watcher->poll("w");
+
+    } elsif ($State eq "ReadingVersionString") {
+	# Read the rest of the version string... 
+    } elsif ($State eq "SetHost") {
+	# Need to request the actual domain get set...
+
+	$Watcher->cb(\&LondWritable);
+	$Watcher->poll("w");
+    } elsif ($State eq "HostSet") {
+	# Reading the 'ok' from the peer.
 
     } elsif ($State eq "RequestingKey") {
 	#  The ok was received.  Now we need to request the key
@@ -784,6 +849,14 @@ sub LondReadable {
     } elsif ($State eq "ReceivingKey") {
 
     } elsif ($State eq "Idle") {
+   
+	# This is as good a spot as any to get the peer version
+	# string:
+   
+	if($LondVersion eq "unknown") {
+	    $LondVersion = $Socket->PeerVersion();
+	    Log("INFO", "Connected to lond version: $LondVersion");
+	}
 	# If necessary, complete a transaction and then go into the
 	# idle queue.
 	#  Note that a trasition to idle indicates a live lond
@@ -800,6 +873,10 @@ sub LondReadable {
 		.$RemoteHost." now ready for action");
 	}
 	ServerToIdle($Socket);	# Next work unit or idle.
+
+	#
+	$LondConnecting = 0;	# Best spot I can think of for this.
+	# 
 	
     } elsif ($State eq "SendingRequest") {
 	#  We need to be writable for this and probably don't belong
@@ -813,7 +890,7 @@ sub LondReadable {
 
 
     } else {
-	 # Invalid state.
+	# Invalid state.
 	Debug(4, "Invalid state in LondReadable");
     }
 }
@@ -895,21 +972,36 @@ sub LondWritable {
 
     SocketDump(6,$Socket);
 
-    if      ($State eq "Connected")         {
-
-	if ($Socket->Writable() != 0) {
-	    #  The write resulted in an error.
-	    # We'll treat this as if the socket got disconnected:
-	    Log("WARNING", "Connection to ".$RemoteHost.
-		" has been disconnected");
+    #  If the socket is writable, we must always write.
+    # Only by writing will we undergo state transitions.
+    # Old logic wrote in state specific code below, however
+    # That forces us at least through another invocation of
+    # this function after writability is possible again.
+    # This logic also factors out common code for handling
+    # write failures... in all cases, write failures 
+    # Kill the socket.
+    #  This logic makes the branches of the >big< if below
+    # so that the writing states are actually NO-OPs.
+
+    if ($Socket->Writable() != 0) {
+	#  The write resulted in an error.
+	# We'll treat this as if the socket got disconnected:
+	Log("WARNING", "Connection to ".$RemoteHost.
+	    " has been disconnected");
+	if(exists($ActiveTransactions{$Socket})) {
 	    FailTransaction($ActiveTransactions{$Socket});
-	    $Watcher->cancel();
-	    KillSocket($Socket);
-	    return;
 	}
-	#  "init" is being sent...
+	$Watcher->cancel();
+	KillSocket($Socket);
+	return;
+    }
 
-	
+
+
+    if      ($State eq "Connected")         {
+
+	#  "init" is being sent...
+ 
     } elsif ($State eq "Initialized")       {
 
 	# Now that init was sent, we switch 
@@ -917,40 +1009,43 @@ sub LondWritable {
 
 	$Watcher->cb(\&LondReadable);
 	$Watcher->poll("r");
-
+	
     } elsif ($State eq "ChallengeReceived") {
 	# We received the challenge, now we 
 	# are echoing it back. This is a no-op,
 	# we're waiting for the state to change
 	
-	if($Socket->Writable() != 0) {
-
-	    $Watcher->cancel();
-	    KillSocket($Socket);
-	    return;
-	}
-	
     } elsif ($State eq "ChallengeReplied")  {
 	# The echo was sent back, so we switch
 	# to watching readability.
 
 	$Watcher->cb(\&LondReadable);
 	$Watcher->poll("r");
+    } elsif ($State eq "RequestingVersion") {
+	# Sending the peer a version request...
+
+    } elsif ($State eq "ReadingVersionString") {
+	# Transition to read since we have sent the
+	# version command and now just need to read the
+	# version string from the peer:
+      
+	$Watcher->cb(\&LondReadable);
+	$Watcher->poll("r");
+      
+    } elsif ($State eq "SetHost") {
+	#  Setting the remote domain...
+
+    } elsif ($State eq "HostSet") {
+	# Back to readable to get the ok.
+      
+	$Watcher->cb(\&LondReadable);
+	$Watcher->poll("r");
+      
 
     } elsif ($State eq "RequestingKey")     {
 	# At this time we're requesting the key.
 	# again, this is essentially a no-op.
-	# we'll write the next chunk until the
-	# state changes.
 
-	if($Socket->Writable() != 0) {
-	    # Write resulted in an error.
-
-	    $Watcher->cancel();
-	    KillSocket($Socket);
-	    return;
-
-	}
     } elsif ($State eq "ReceivingKey")      {
 	# Now we need to wait for the key
 	# to come back from the peer:
@@ -959,20 +1054,10 @@ sub LondWritable {
 	$Watcher->poll("r");
 
     } elsif ($State eq "SendingRequest")    {
+ 
 	# At this time we are sending a request to the
 	# peer... write the next chunk:
 
-	if($Socket->Writable() != 0) {
-
-	    if(exists($ActiveTransactions{$Socket})) {
-		Debug(3, "Lond connection lost, failing transactions");
-		FailTransaction($ActiveTransactions{$Socket});
-	    }
-	    $Watcher->cancel();
-	    KillSocket($Socket);
-	    return;
-	    
-	}
 
     } elsif ($State eq "ReceivingReply")    {
 	# The send has completed.  Wait for the
@@ -1065,7 +1150,9 @@ sub MakeLondConnection {
 			   data     => $Connection,
 			   desc => 'Connection to lond server');
 	$ActiveConnections{$Connection} = $event;
-	
+	if ($ConnectionCount == 0) {
+	    &SetupTimer;	# Need to handle timeouts with connections...
+	}
 	$ConnectionCount++;
 	Debug(4, "Connection count = ".$ConnectionCount);
 	if($ConnectionCount == 1) { # First Connection:
@@ -1073,6 +1160,7 @@ sub MakeLondConnection {
 	}
 	Log("SUCESS", "Created connection ".$ConnectionCount
 	    ." to host ".GetServerHost());
+	$LondConnecting = 1;	# Connection in progress.
 	return 1;		# Return success.
     }
     
@@ -1104,8 +1192,8 @@ The text of the request to send.
 =cut
 
 sub StartRequest {
-    my $Lond     = shift;
-    my $Request  = shift;	# This is a LondTransaction.
+
+    my ($Lond, $Request) = @_;
     
     Debug(6, "StartRequest: ".$Request->getRequest());
 
@@ -1156,15 +1244,17 @@ sub QueueTransaction {
     if(!defined $LondSocket) {	# Need to queue request.
 	Debug(5,"Must queue...");
 	$WorkQueue->enqueue($requestData);
-	if($ConnectionCount < $MaxConnectionCount) {
+	if(($ConnectionCount < $MaxConnectionCount) && !$LondConnecting) {
 	    if($ConnectionRetriesLeft > 0) {
 		Debug(5,"Starting additional lond connection");
 		if(MakeLondConnection() == 0) {
 		    EmptyQueue();	# Fail transactions, can't make connection.
+		    CloseAllLondConnections; # Should all be closed but...
 		}
 	    } else {
 		ShowStatus(GetServerHost()." >>> DEAD !!!! <<<");
 		EmptyQueue();	# It's worse than that ... he's dead Jim.
+		CloseAllLondConnections; # Should all be closed but..
 	    }
 	}
     } else {			# Can start the request:
@@ -1207,7 +1297,7 @@ sub ClientRequest {
     Debug(8,"Data: ".$data." this read: ".$thisread);
     $data = $data.$thisread;	# Append new data.
     $watcher->data($data);
-    if($data =~ /(.*\n)/) {	# Request entirely read.
+    if($data =~ /\n$/) {	# Request entirely read.
 	if($data eq "close_connection_exit\n") {
 	    Log("CRITICAL",
 		"Request Close Connection ... exiting");
@@ -1215,6 +1305,9 @@ sub ClientRequest {
 	    exit;
 	}
 	Debug(8, "Complete transaction received: ".$data);
+	if($LogTransactions) {
+	    Log("SUCCESS", "Transaction: '$data'"); # Transaction has \n.
+	}
 	my $Transaction = LondTransaction->new($data);
 	$Transaction->SetClient($socket);
 	QueueTransaction($Transaction);
@@ -1313,7 +1406,7 @@ sub SetupLoncListener {
     my $SocketName = GetLoncSocketPath();
     unlink($SocketName);
     unless ($socket =IO::Socket::UNIX->new(Local  => $SocketName,
-					    Listen => 10, 
+					    Listen => 250, 
 					    Type   => SOCK_STREAM)) {
 	die "Failed to create a lonc listner socket";
     }
@@ -1323,6 +1416,24 @@ sub SetupLoncListener {
 	      fd     => $socket);
 }
 
+#
+#   Toggle transaction logging.
+#  Implicit inputs:  
+#     LogTransactions
+#  Implicit Outputs:
+#     LogTransactions
+sub ToggleTransactionLogging {
+    print STDERR "Toggle transaction logging...\n";
+    if(!$LogTransactions) {
+	$LogTransactions = 1;
+    } else {
+	$LogTransactions = 0;
+    }
+
+
+    Log("SUCCESS", "Toggled transaction logging: $LogTransactions \n");
+}
+
 =pod 
 
 =head2 ChildStatus
@@ -1333,6 +1444,7 @@ into the status file.
 We also use this to reset the retries count in order to allow the
 client to retry connections with a previously dead server.
 =cut
+
 sub ChildStatus {
     my $event = shift;
     my $watcher = $event->w;
@@ -1342,6 +1454,22 @@ sub ChildStatus {
     my $fh = IO::File->new(">>$docdir/lon-status/loncstatus.txt");
     print $fh $$."\t".$RemoteHost."\t".$Status."\t".
 	$RecentLogEntry."\n";
+    #
+    #  Write out information about each of the connections:
+    #
+    if ($DebugLevel > 2) {
+	print $fh "Active connection statuses: \n";
+	my $i = 1;
+	print STDERR  "================================= Socket Status Dump:\n";
+	foreach my $item (keys %ActiveConnections) {
+	    my $Socket = $ActiveConnections{$item}->data;
+	    my $state  = $Socket->GetState();
+	    print $fh "Connection $i State: $state\n";
+	    print STDERR "---------------------- Connection $i \n";
+	    $Socket->Dump(-1);	# Ensure it gets dumped..
+	    $i++;	
+	}
+    }
     $ConnectionRetriesLeft = $ConnectionRetries;
 }
 
@@ -1406,11 +1534,12 @@ sub ChildProcess {
     Event->signal(signal   => "USR1",
 		  cb       => \&ChildStatus,
 		  data     => "USR1");
+    Event->signal(signal   => "USR2",
+		  cb       => \&ToggleTransactionLogging);
     Event->signal(signal   => "INT",
 		  cb       => \&ToggleDebug,
 		  data     => "INT");
 
-    SetupTimer();
     
     SetupLoncListener();
     
@@ -1433,9 +1562,10 @@ sub ChildProcess {
 #  Create a new child for host passed in:
 
 sub CreateChild {
+    my $host = shift;
+
     my $sigset = POSIX::SigSet->new(SIGINT);
     sigprocmask(SIG_BLOCK, $sigset);
-    my $host = shift;
     $RemoteHost = $host;
     Log("CRITICAL", "Forking server for ".$host);
     my $pid          = fork;
@@ -1706,17 +1836,33 @@ sub KillThemAll {
     local($SIG{CHLD}) = 'IGNORE';      # Our children >will< die.
     foreach my $pid (keys %ChildHash) {
 	my $serving = $ChildHash{$pid};
-	Debug(2, "Killing lonc for $serving pid = $pid");
-	ShowStatus("Killing lonc for $serving pid = $pid");
-	Log("CRITICAL", "Killing lonc for $serving pid = $pid");
+	ShowStatus("Nicely Killing lonc for $serving pid = $pid");
+	Log("CRITICAL", "Nicely Killing lonc for $serving pid = $pid");
 	kill 'QUIT' => $pid;
-	delete($ChildHash{$pid});
     }
-    my $execdir = $perlvar{'lonDaemons'};
-    unlink("$execdir/logs/lonc.pid");
+
 
 }
 
+
+#
+#  Kill all children via KILL.  Just in case the
+#  first shot didn't get them.
+
+sub really_kill_them_all_dammit
+{
+    Debug(2, "Kill them all Dammit");
+    local($SIG{CHLD} = 'IGNORE'); # In case some purist reenabled them.
+    foreach my $pid (keys %ChildHash) {
+	my $serving = $ChildHash{$pid};
+	&ShowStatus("Nastily killing lonc for $serving pid = $pid");
+	Log("CRITICAL", "Nastily killing lonc for $serving pid = $pid");
+	kill 'KILL' => $pid;
+	delete($ChildHash{$pid});
+	my $execdir = $perlvar{'lonDaemons'};
+	unlink("$execdir/logs/lonc.pid");
+    }
+}
 =pod
 
 =head1 Terminate
@@ -1726,7 +1872,15 @@ Terminate the system.
 =cut
 
 sub Terminate {
-    KillThemAll;
+    &Log("CRITICAL", "Asked to kill children.. first be nice...");
+    &KillThemAll;
+    #
+    #  By now they really should all be dead.. but just in case 
+    #  send them all SIGKILL's after a bit of waiting:
+
+    sleep(4);
+    &Log("CRITICAL", "Now kill children nasty");
+    &really_kill_them_all_dammit;
     Log("CRITICAL","Master process exiting");
     exit 0;