--- loncom/lond	2007/03/28 20:28:29	1.364
+++ loncom/lond	2008/04/16 22:51:21	1.399
@@ -2,7 +2,7 @@
 # The LearningOnline Network
 # lond "LON Daemon" Server (port "LOND" 5663)
 #
-# $Id: lond,v 1.364 2007/03/28 20:28:29 albertel Exp $
+# $Id: lond,v 1.399 2008/04/16 22:51:21 raeburn Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -49,17 +49,17 @@ use localenroll;
 use localstudentphoto;
 use File::Copy;
 use File::Find;
-use LONCAPA::ConfigFileEdit;
 use LONCAPA::lonlocal;
 use LONCAPA::lonssl;
 use Fcntl qw(:flock);
+use Apache::lonnet;
 
 my $DEBUG = 0;		       # Non zero to enable debug log entries.
 
 my $status='';
 my $lastlog='';
 
-my $VERSION='$Revision: 1.364 $'; #' stupid emacs
+my $VERSION='$Revision: 1.399 $'; #' stupid emacs
 my $remoteVERSION;
 my $currenthostid="default";
 my $currentdomainid;
@@ -69,7 +69,6 @@ my $clientip;			# IP address of client.
 my $clientname;			# LonCAPA name of client.
 
 my $server;
-my $thisserver;			# DNS of us.
 
 my $keymode;
 
@@ -85,12 +84,6 @@ my $tmpsnum = 0;		# Id of tmpputs.
 
 my $ConnectionType;
 
-my %hostid;			# ID's for hosts in cluster by ip.
-my %hostdom;			# LonCAPA domain for hosts in cluster.
-my %hostname;			# DNSname -> ID's mapping.
-my %hostip;			# IPs for hosts in cluster.
-my %hostdns;			# ID's of hosts looked up by DNS name.
-
 my %managers;			# Ip -> manager names
 
 my %perlvar;			# Will have the apache conf defined perl vars.
@@ -142,7 +135,7 @@ my @adderrors    = ("ok",
 		    "lcuseradd Unable to make www member of users's group",
 		    "lcuseradd Unable to su to root",
 		    "lcuseradd Unable to set password",
-		    "lcuseradd Usrname has invalid characters",
+		    "lcuseradd Username has invalid characters",
 		    "lcuseradd Password has an invalid character",
 		    "lcuseradd User already exists",
 		    "lcuseradd Could not add user.",
@@ -178,19 +171,16 @@ sub ResetStatistics {
 #   $Socket      - Socket open on client.
 #   $initcmd     - The full text of the init command.
 #
-# Implicit inputs:
-#    $thisserver - Our DNS name.
-#
 # Returns:
 #     IDEA session key on success.
 #     undef on failure.
 #
 sub LocalConnection {
     my ($Socket, $initcmd) = @_;
-    Debug("Attempting local connection: $initcmd client: $clientip me: $thisserver");
+    Debug("Attempting local connection: $initcmd client: $clientip");
     if($clientip ne "127.0.0.1") {
 	&logthis('<font color="red"> LocalConnection rejecting non local: '
-		 ."$clientip ne $thisserver </font>");
+		 ."$clientip ne 127.0.0.1 </font>");
 	close $Socket;
 	return undef;
     }  else {
@@ -424,7 +414,7 @@ sub ReadManagerTable {
       if ($host =~ "^#") {                  # Comment line.
          next;
       }
-      if (!defined $hostip{$host}) { # This is a non cluster member
+      if (!defined &Apache::lonnet::get_host_ip($host)) { # This is a non cluster member
 	    #  The entry is of the form:
 	    #    cluname:hostname
 	    #  cluname - A 'cluster hostname' is needed in order to negotiate
@@ -442,7 +432,7 @@ sub ReadManagerTable {
          }
       } else {
          logthis('<font color="green"> existing host'." $host</font>\n");
-         $managers{$hostip{$host}} = $host;  # Use info from cluster tab if clumemeber
+         $managers{&Apache::lonnet::get_host_ip($host)} = $host;  # Use info from cluster tab if clumemeber
       }
    }
 }
@@ -1006,7 +996,7 @@ sub ping_handler {
     my ($cmd, $tail, $client) = @_;
     Debug("$cmd $tail $client .. $currenthostid:");
    
-    Reply( $client,"$currenthostid\n","$cmd:$tail");
+    Reply( $client,\$currenthostid,"$cmd:$tail");
    
     return 1;
 }
@@ -1032,7 +1022,7 @@ sub ping_handler {
 sub pong_handler {
     my ($cmd, $tail, $replyfd) = @_;
 
-    my $reply=&reply("ping",$clientname);
+    my $reply=&Apache::lonnet::reply("ping",$clientname);
     &Reply( $replyfd, "$currenthostid:$reply\n", "$cmd:$tail"); 
     return 1;
 }
@@ -1076,7 +1066,7 @@ sub establish_key_handler {
     $key=substr($key,0,32);
     my $cipherkey=pack("H32",$key);
     $cipher=new IDEA $cipherkey;
-    &Reply($replyfd, "$buildkey\n", "$cmd:$tail"); 
+    &Reply($replyfd, \$buildkey, "$cmd:$tail"); 
    
     return 1;
 
@@ -1113,7 +1103,7 @@ sub load_handler {
    
     my $loadpercent=100*$loadavg/$perlvar{'lonLoadLim'};
 
-    &Reply( $replyfd, "$loadpercent\n", "$cmd:$tail");
+    &Reply( $replyfd, \$loadpercent, "$cmd:$tail");
    
     return 1;
 }
@@ -1142,8 +1132,8 @@ sub load_handler {
 sub user_load_handler {
     my ($cmd, $tail, $replyfd) = @_;
 
-    my $userloadpercent=&userload();
-    &Reply($replyfd, "$userloadpercent\n", "$cmd:$tail");
+    my $userloadpercent=&Apache::lonnet::userload();
+    &Reply($replyfd, \$userloadpercent, "$cmd:$tail");
     
     return 1;
 }
@@ -1186,7 +1176,7 @@ sub user_authorization_type {
 	} else {
             $type .= ':';
         }
-	&Reply( $replyfd, "$type\n", $userinput);
+	&Reply( $replyfd, \$type, $userinput);
     }
   
     return 1;
@@ -1222,7 +1212,7 @@ sub push_file_handler {
 	# process making the request.
       
 	my $reply = &PushFile($userinput);
-	&Reply($client, "$reply\n", $userinput);
+	&Reply($client, \$reply, $userinput);
 
     } else {
 	&Failure( $client, "refused\n", $userinput);
@@ -1231,8 +1221,10 @@ sub push_file_handler {
 }
 &register_handler("pushfile", \&push_file_handler, 1, 0, 1);
 
+# The du_handler routine should be considered obsolete and is retained
+# for communication with legacy servers.  Please see the du2_handler.
 #
-#   du  - list the disk usuage of a directory recursively. 
+#   du  - list the disk usage of a directory recursively. 
 #    
 #   note: stolen code from the ls file handler
 #   under construction by Rick Banghart 
@@ -1274,7 +1266,7 @@ sub du_handler {
 	chdir($ududir);
 	find($code,$ududir);
 	$total_size=int($total_size/1024);
-	&Reply($client,"$total_size\n","$cmd:$ududir");
+	&Reply($client,\$total_size,"$cmd:$ududir");
     } else {
 	&Failure($client, "bad_directory:$ududir\n","$cmd:$ududir"); 
     }
@@ -1282,9 +1274,73 @@ sub du_handler {
 }
 &register_handler("du", \&du_handler, 0, 1, 0);
 
+# Please also see the du_handler, which is obsoleted by du2. 
+# du2_handler differs from du_handler in that required path to directory
+# provided by &propath() is prepended in the handler instead of on the 
+# client side.
 #
-# The ls_handler routine should be considered obosolete and is retained
-# for communication with legacy servers.  Please see the ls2_handler.
+#   du2  - list the disk usage of a directory recursively.
+#
+# Parameters:
+#    $cmd        - The command that dispatched us (du).
+#    $tail       - The tail of the request that invoked us.
+#                  $tail is a : separated list of the following:
+#                   - $ududir - directory path to list (before prepending)
+#                   - $getpropath = 1 if &propath() should prepend
+#                   - $uname - username to use for &propath or user dir
+#                   - $udom - domain to use for &propath or user dir
+#                   All are escaped.
+#    $client     - Socket open on the client.
+# Returns:
+#     1 - indicating that the daemon should not disconnect.
+# Side Effects:
+#   The reply is written to $client.
+#
+
+sub du2_handler {
+    my ($cmd, $tail, $client) = @_;
+    my ($ududir,$getpropath,$uname,$udom) = map { &unescape($_) } (split(/:/, $tail));
+    my $userinput = "$cmd:$tail";
+    if (($ududir=~/\.\./) || (($ududir!~m|^/home/httpd/|) && (!$getpropath))) {
+        &Failure($client,"refused\n","$cmd:$tail");
+        return 1;
+    }
+    if ($getpropath) {
+        if (($uname =~ /^$LONCAPA::match_name$/) && ($udom =~ /^$LONCAPA::match_domain$/)) {
+            $ududir = &propath($udom,$uname).'/'.$ududir;
+        } else {
+            &Failure($client,"refused\n","$cmd:$tail");
+            return 1;
+        }
+    }
+    #  Since $ududir could have some nasties in it,
+    #  we will require that ududir is a valid
+    #  directory.  Just in case someone tries to
+    #  slip us a  line like .;(cd /home/httpd rm -rf*)
+    #  etc.
+    #
+    if (-d $ududir) {
+        my $total_size=0;
+        my $code=sub {
+            if ($_=~/\.\d+\./) { return;}
+            if ($_=~/\.meta$/) { return;}
+            if (-d $_)         { return;}
+            $total_size+=(stat($_))[7];
+        };
+        chdir($ududir);
+        find($code,$ududir);
+        $total_size=int($total_size/1024);
+        &Reply($client,\$total_size,"$cmd:$ududir");
+    } else {
+        &Failure($client, "bad_directory:$ududir\n","$cmd:$tail");
+    }
+    return 1;
+}
+&register_handler("du2", \&du2_handler, 0, 1, 0);
+
+#
+# The ls_handler routine should be considered obsolete and is retained
+# for communication with legacy servers.  Please see the ls3_handler.
 #
 #   ls  - list the contents of a directory.  For each file in the
 #    selected directory the filename followed by the full output of
@@ -1343,15 +1399,16 @@ sub ls_handler {
 	$ulsout='no_such_dir';
     }
     if ($ulsout eq '') { $ulsout='empty'; }
-    &Reply($client, "$ulsout\n", $userinput); # This supports debug logging.
+    &Reply($client, \$ulsout, $userinput); # This supports debug logging.
     
     return 1;
 
 }
 &register_handler("ls", \&ls_handler, 0, 1, 0);
 
-#
-# Please also see the ls_handler, which this routine obosolets.
+# The ls2_handler routine should be considered obsolete and is retained
+# for communication with legacy servers.  Please see the ls3_handler.
+# Please also see the ls_handler, which was itself obsoleted by ls2.
 # ls2_handler differs from ls_handler in that it escapes its return 
 # values before concatenating them together with ':'s.
 #
@@ -1412,10 +1469,119 @@ sub ls2_handler {
         $ulsout='no_such_dir';
    }
    if ($ulsout eq '') { $ulsout='empty'; }
-   &Reply($client, "$ulsout\n", $userinput); # This supports debug logging.
+   &Reply($client, \$ulsout, $userinput); # This supports debug logging.
    return 1;
 }
 &register_handler("ls2", \&ls2_handler, 0, 1, 0);
+#
+#   ls3  - list the contents of a directory.  For each file in the
+#    selected directory the filename followed by the full output of
+#    the stat function is returned.  The returned info for each
+#    file are separated by ':'.  The stat fields are separated by &'s.
+# Parameters:
+#    $cmd        - The command that dispatched us (ls).
+#    $tail       - The tail of the request that invoked us.
+#                  $tail is a : separated list of the following:
+#                   - $ulsdir - directory path to list (before prepending)
+#                   - $getpropath = 1 if &propath() should prepend
+#                   - $getuserdir = 1 if path to user dir in lonUsers should
+#                                     prepend
+#                   - $alternate_root - path to prepend
+#                   - $uname - username to use for &propath or user dir
+#                   - $udom - domain to use for &propath or user dir
+#            All of these except $getpropath and &getuserdir are escaped.    
+#                  no_such_dir.
+#    $client     - Socket open on the client.
+# Returns:
+#     1 - indicating that the daemon should not disconnect.
+# Side Effects:
+#   The reply is written to $client.
+#
+
+sub ls3_handler {
+    my ($cmd, $tail, $client) = @_;
+    my $userinput = "$cmd:$tail";
+    my ($ulsdir,$getpropath,$getuserdir,$alternate_root,$uname,$udom) =
+        split(/:/,$tail);
+    if (defined($ulsdir)) {
+        $ulsdir = &unescape($ulsdir);
+    }
+    if (defined($alternate_root)) {
+        $alternate_root = &unescape($alternate_root);
+    }
+    if (defined($uname)) {
+        $uname = &unescape($uname);
+    }
+    if (defined($udom)) {
+        $udom = &unescape($udom);
+    }
+
+    my $dir_root = $perlvar{'lonDocRoot'};
+    if ($getpropath) {
+        if (($uname =~ /^$LONCAPA::match_name$/) && ($udom =~ /^$LONCAPA::match_domain$/)) {
+            $dir_root = &propath($udom,$uname);
+            $dir_root =~ s/\/$//;
+        } else {
+            &Failure($client,"refused\n","$cmd:$tail");
+            return 1;
+        }
+    } elsif ($getuserdir) {
+        if (($uname =~ /^$LONCAPA::match_name$/) && ($udom =~ /^$LONCAPA::match_domain$/)) {
+            my $subdir=$uname.'__';
+            $subdir =~ s/(.)(.)(.).*/$1\/$2\/$3/;
+            $dir_root = $Apache::lonnet::perlvar{'lonUsersDir'}
+                       ."/$udom/$subdir/$uname";
+        } else {
+            &Failure($client,"refused\n","$cmd:$tail");
+            return 1;
+        }
+    } elsif (defined($alternate_root)) {
+        $dir_root = $alternate_root;
+    }
+    if (defined($dir_root)) {
+        $ulsdir = $dir_root.'/'.$ulsdir;
+    }
+    my $obs;
+    my $rights;
+    my $ulsout='';
+    my $ulsfn;
+    if (-e $ulsdir) {
+        if(-d $ulsdir) {
+            if (opendir(LSDIR,$ulsdir)) {
+                while ($ulsfn=readdir(LSDIR)) {
+                    undef($obs);
+                    undef($rights);
+                    my @ulsstats=stat($ulsdir.'/'.$ulsfn);
+                    #We do some obsolete checking here
+                    if(-e $ulsdir.'/'.$ulsfn.".meta") {
+                        open(FILE, $ulsdir.'/'.$ulsfn.".meta");
+                        my @obsolete=<FILE>;
+                        foreach my $obsolete (@obsolete) {
+                            if($obsolete =~ m/(<obsolete>)(on|1)/) { $obs = 1; }
+                            if($obsolete =~ m|(<copyright>)(default)|) {
+                                $rights = 1;
+                            }
+                        }
+                    }
+                    my $tmp = $ulsfn.'&'.join('&',@ulsstats);
+                    if ($obs    eq '1') { $tmp.="&1"; } else { $tmp.="&0"; }
+                    if ($rights eq '1') { $tmp.="&1"; } else { $tmp.="&0"; }
+                    $ulsout.= &escape($tmp).':';
+                }
+                closedir(LSDIR);
+            }
+        } else {
+            my @ulsstats=stat($ulsdir);
+            $ulsout.=$ulsfn.'&'.join('&',@ulsstats).':';
+        }
+    } else {
+        $ulsout='no_such_dir';
+   }
+   if ($ulsout eq '') { $ulsout='empty'; }
+   &Reply($client, \$ulsout, $userinput); # This supports debug logging.
+   return 1;
+}
+&register_handler("ls3", \&ls3_handler, 0, 1, 0);
 
 #   Process a reinit request.  Reinit requests that either
 #   lonc or lond be reinitialized so that an updated 
@@ -1440,7 +1606,7 @@ sub reinit_process_handler {
     if(&ValidManager($cert)) {
 	chomp($userinput);
 	my $reply = &ReinitProcess($userinput);
-	&Reply( $client,  "$reply\n", $userinput);
+	&Reply( $client,  \$reply, $userinput);
     } else {
 	&Failure( $client, "refused\n", $userinput);
     }
@@ -1524,13 +1690,15 @@ sub authenticate_handler {
     #  udom    - User's domain.
     #  uname   - Username.
     #  upass   - User's password.
+    #  checkdefauth - Pass to validate_user() to try authentication
+    #                 with default auth type(s) if no user account.
     
-    my ($udom,$uname,$upass)=split(/:/,$tail);
-    &Debug(" Authenticate domain = $udom, user = $uname, password = $upass");
+    my ($udom, $uname, $upass, $checkdefauth)=split(/:/,$tail);
+    &Debug(" Authenticate domain = $udom, user = $uname, password = $upass,  checkdefauth = $checkdefauth");
     chomp($upass);
     $upass=&unescape($upass);
 
-    my $pwdcorrect = &validate_user($udom, $uname, $upass);
+    my $pwdcorrect = &validate_user($udom,$uname,$upass,$checkdefauth);
     if($pwdcorrect) {
 	&Reply( $client, "authorized\n", $userinput);
 	#
@@ -1615,7 +1783,7 @@ sub change_password_handler {
 	    my $result = &change_unix_password($uname, $npass);
 	    &logthis("Result of password change for $uname: ".
 		     $result);
-	    &Reply($client, "$result\n", $userinput);
+	    &Reply($client, \$result, $userinput);
 	} else {
 	    # this just means that the current password mode is not
 	    # one we know how to change (e.g the kerberos auth modes or
@@ -1676,9 +1844,9 @@ sub add_user_handler {
 	    }
 	    unless ($fperror) {
 		my $result=&make_passwd_file($uname, $umode,$npass, $passfilename);
-		&Reply($client, $result, $userinput);     #BUGBUG - could be fail
+		&Reply($client,\$result, $userinput);     #BUGBUG - could be fail
 	    } else {
-		&Failure($client, "$fperror\n", $userinput);
+		&Failure($client, \$fperror, $userinput);
 	    }
 	}
 	umask($oldumask);
@@ -1745,9 +1913,9 @@ sub change_authentication_handler {
 		my $result = &change_unix_password($uname, $npass);
 		&logthis("Result of password change for $uname: ".$result);
 		if ($result eq "ok") {
-		    &Reply($client, "$result\n")
+		    &Reply($client, \$result);
 		} else {
-		    &Failure($client, "$result\n");
+		    &Failure($client, \$result);
 		}
 	    } else {
 		my $result=&make_passwd_file($uname, $umode,$npass,$passfilename);
@@ -1766,7 +1934,7 @@ sub change_authentication_handler {
 			&manage_permissions("/$udom/_au", $udom, $uname, "$umode:");
 		    }
 		}
-		&Reply($client, $result, $userinput);
+		&Reply($client, \$result, $userinput);
 	    }
 	       
 
@@ -1850,13 +2018,13 @@ sub update_resource_handler {
 	    my $now=time;
 	    my $since=$now-$atime;
 	    if ($since>$perlvar{'lonExpire'}) {
-		my $reply=&reply("unsub:$fname","$clientname");
+		my $reply=&Apache::lonnet::reply("unsub:$fname","$clientname");
 		&devalidate_meta_cache($fname);
 		unlink("$fname");
 		unlink("$fname.meta");
 	    } else {
 		my $transname="$fname.in.transfer";
-		my $remoteurl=&reply("sub:$fname","$clientname");
+		my $remoteurl=&Apache::lonnet::reply("sub:$fname","$clientname");
 		my $response;
 		alarm(120);
 		{
@@ -1901,22 +2069,12 @@ sub devalidate_meta_cache {
     my ($url) = @_;
     use Cache::Memcached;
     my $memcache = new Cache::Memcached({'servers'=>['127.0.0.1:11211']});
-    $url = &declutter($url);
+    $url = &Apache::lonnet::declutter($url);
     $url =~ s-\.meta$--;
     my $id = &escape('meta:'.$url);
     $memcache->delete($id);
 }
 
-sub declutter {
-    my $thisfn=shift;
-    $thisfn=~s/^\Q$perlvar{'lonDocRoot'}\E//;
-    $thisfn=~s/^\///;
-    $thisfn=~s|^adm/wrapper/||;
-    $thisfn=~s|^adm/coursedocs/showdoc/||;
-    $thisfn=~s/^res\///;
-    $thisfn=~s/\?.+$//;
-    return $thisfn;
-}
 #
 #   Fetch a user file from a remote server to the user's home directory
 #   userfiles subdir.
@@ -2115,6 +2273,37 @@ sub rename_user_file_handler {
 &register_handler("renameuserfile", \&rename_user_file_handler, 0,1,0);
 
 #
+#  Checks if the specified user has an active session on the server
+#  return ok if so, not_found if not
+#
+# Parameters:
+#   cmd      - The request keyword that dispatched to tus.
+#   tail     - The tail of the request (colon separated parameters).
+#   client   - Filehandle open on the client.
+# Return:
+#    1.
+sub user_has_session_handler {
+    my ($cmd, $tail, $client) = @_;
+
+    my ($udom, $uname) = map { &unescape($_) } (split(/:/, $tail));
+    
+    &logthis("Looking for $udom $uname");
+    opendir(DIR,$perlvar{'lonIDsDir'});
+    my $filename;
+    while ($filename=readdir(DIR)) {
+	last if ($filename=~/^\Q$uname\E_\d+_\Q$udom\E_/);
+    }
+    if ($filename) {
+	&Reply($client, "ok\n", "$cmd:$tail");
+    } else {
+	&Failure($client, "not_found\n", "$cmd:$tail");
+    }
+    return 1;
+
+}
+&register_handler("userhassession", \&user_has_session_handler, 0,1,0);
+
+#
 #  Authenticate access to a user file by checking that the token the user's 
 #  passed also exists in their session file
 #
@@ -2130,24 +2319,24 @@ sub token_auth_user_file_handler {
     my ($fname, $session) = split(/:/, $tail);
     
     chomp($session);
-    my $reply="non_auth\n";
+    my $reply="non_auth";
     my $file = $perlvar{'lonIDsDir'}.'/'.$session.'.id';
     if (open(ENVIN,"$file")) {
 	flock(ENVIN,LOCK_SH);
 	tie(my %disk_env,'GDBM_File',"$file",&GDBM_READER(),0640);
 	if (exists($disk_env{"userfile.$fname"})) {
-	    $reply="ok\n";
+	    $reply="ok";
 	} else {
 	    foreach my $envname (keys(%disk_env)) {
 		if ($envname=~ m|^userfile\.\Q$fname\E|) {
-		    $reply="ok\n";
+		    $reply="ok";
 		    last;
 		}
 	    }
 	}
 	untie(%disk_env);
 	close(ENVIN);
-	&Reply($client, $reply, "$cmd:$tail");
+	&Reply($client, \$reply, "$cmd:$tail");
     } else {
 	&Failure($client, "invalid_token\n", "$cmd:$tail");
     }
@@ -2207,13 +2396,13 @@ sub subscribe_handler {
 &register_handler("sub", \&subscribe_handler, 0, 1, 0);
 
 #
-#   Determine the version of a resource (?) Or is it return
-#   the top version of the resource?  Not yet clear from the
-#   code in currentversion.
+#   Determine the latest version of a resource (it looks for the highest
+#   past version and then returns that +1)
 #
 # Parameters:
 #    $cmd      - The command that got us here.
 #    $tail     - Tail of the command (remaining parameters).
+#                 (Should consist of an absolute path to a file)
 #    $client   - File descriptor connected to client.
 # Returns
 #     0        - Requested to exit, caller should shut down.
@@ -2571,10 +2760,11 @@ sub get_profile_entry {
     my ($udom,$uname,$namespace,$what) = split(/:/,$tail);
     chomp($what);
 
+
     my $replystring = read_profile($udom, $uname, $namespace, $what);
     my ($first) = split(/:/,$replystring);
     if($first ne "error") {
-	&Reply($client, "$replystring\n", $userinput);
+	&Reply($client, \$replystring, $userinput);
     } else {
 	&Failure($client, $replystring." while attempting get\n", $userinput);
     }
@@ -2714,7 +2904,7 @@ sub get_profile_keys {
 	}
 	if (&untie_user_hash($hashref)) {
 	    $qresult=~s/\&$//;
-	    &Reply($client, "$qresult\n", $userinput);
+	    &Reply($client, \$qresult, $userinput);
 	} else {
 	    &Failure($client, "error: ".($!+0)." untie(GDBM) Failed ".
 		    "while attempting keys\n", $userinput);
@@ -2784,7 +2974,7 @@ sub dump_profile_database {
 		}
 	    }
 	    chop($qresult);
-	    &Reply($client , "$qresult\n", $userinput);
+	    &Reply($client , \$qresult, $userinput);
 	} else {
 	    &Failure( $client, "error: ".($!+0)." untie(GDBM) Failed ".
 		     "while attempting currentdump\n", $userinput);
@@ -2867,7 +3057,7 @@ sub dump_with_regexp {
 	}
 	if (&untie_user_hash($hashref)) {
 	    chop($qresult);
-	    &Reply($client, "$qresult\n", $userinput);
+	    &Reply($client, \$qresult, $userinput);
 	} else {
 	    &Failure( $client, "error: ".($!+0)." untie(GDBM) Failed ".
 		     "while attempting dump\n", $userinput);
@@ -3075,7 +3265,7 @@ sub restore_handler {
 	}
 	if (&untie_user_hash($hashref)) {
 	    $qresult=~s/\&$//;
-	    &Reply( $client, "$qresult\n", $userinput);
+	    &Reply( $client, \$qresult, $userinput);
 	} else {
 	    &Failure($client, "error: ".($!+0)." untie(GDBM) Failed ".
 		    "while attempting restore\n", $userinput);
@@ -3156,7 +3346,7 @@ sub retrieve_chat_handler {
 	$reply.=&escape($_).':';
     }
     $reply=~s/\:$//;
-    &Reply($client, $reply."\n", $userinput);
+    &Reply($client, \$reply, $userinput);
 
 
     return 1;
@@ -3293,6 +3483,22 @@ sub put_course_id_handler {
 	foreach my $pair (@pairs) {
             my ($key,$courseinfo) = split(/=/,$pair,2);
             $courseinfo =~ s/=/:/g;
+            if (defined($hashref->{$key})) {
+                my $value = &Apache::lonnet::thaw_unescape($hashref->{$key});
+                if (ref($value) eq 'HASH') {
+                    my @items = ('description','inst_code','owner','type');
+                    my @new_items = split(/:/,$courseinfo,-1);
+                    my %storehash; 
+                    for (my $i=0; $i<@new_items; $i++) {
+                        $storehash{$items[$i]} = &unescape($new_items[$i]);
+                    }
+                    $hashref->{$key} = 
+                        &Apache::lonnet::freeze_escape(\%storehash);
+                    my $unesc_key = &unescape($key);
+                    $hashref->{&escape('lasttime:'.$unesc_key)} = $now;
+                    next;
+                }
+            }
             my @current_items = split(/:/,$hashref->{$key},-1);
             shift(@current_items); # remove description
             pop(@current_items);   # remove last access
@@ -3309,7 +3515,7 @@ sub put_course_id_handler {
                     }
                 }
             }
-	    $hashref->{$key}=$courseinfo.':'.$now;
+            $hashref->{$key}=$courseinfo.':'.$now;
 	}
 	if (&untie_domain_hash($hashref)) {
 	    &Reply( $client, "ok\n", $userinput);
@@ -3323,12 +3529,54 @@ sub put_course_id_handler {
 		 ." tie(GDBM) Failed ".
 		 "while attempting courseidput\n", $userinput);
     }
-    
 
     return 1;
 }
 &register_handler("courseidput", \&put_course_id_handler, 0, 1, 0);
 
+sub put_course_id_hash_handler {
+    my ($cmd, $tail, $client) = @_;
+    my $userinput = "$cmd:$tail";
+    my ($udom,$mode,$what) = split(/:/, $tail,3);
+    chomp($what);
+    my $now=time;
+    my @pairs=split(/\&/,$what);
+    my $hashref = &tie_domain_hash($udom, "nohist_courseids", &GDBM_WRCREAT());
+    if ($hashref) {
+        foreach my $pair (@pairs) {
+            my ($key,$value)=split(/=/,$pair);
+            my $unesc_key = &unescape($key);
+            if ($mode ne 'timeonly') {
+                if (!defined($hashref->{&escape('lasttime:'.$unesc_key)})) {
+                    my $curritems = &Apache::lonnet::thaw_unescape($key); 
+                    if (ref($curritems) ne 'HASH') {
+                        my @current_items = split(/:/,$hashref->{$key},-1);
+                        my $lasttime = pop(@current_items);
+                        $hashref->{&escape('lasttime:'.$unesc_key)} = $lasttime;
+                    } else {
+                        $hashref->{&escape('lasttime:'.$unesc_key)} = '';
+                    }
+                } 
+                $hashref->{$key} = $value;
+            }
+            if ($mode ne 'notime') {
+                $hashref->{&escape('lasttime:'.$unesc_key)} = $now;
+            }
+        }
+        if (&untie_domain_hash($hashref)) {
+            &Reply($client, "ok\n", $userinput);
+        } else {
+            &Failure($client, "error: ".($!+0)." untie(GDBM) Failed ".
+                     "while attempting courseidputhash\n", $userinput);
+        }
+    } else {
+        &Failure( $client, "error: ".($!+0)." tie(GDBM) Failed ".
+                  "while attempting courseidputhash\n", $userinput);
+    }
+    return 1;
+}
+&register_handler("courseidputhash", \&put_course_id_hash_handler, 0, 1, 0);
+
 #  Retrieves the value of a course id resource keyword pattern
 #  defined since a starting date.  Both the starting date and the
 #  keyword pattern are optional.  If the starting date is not supplied it
@@ -3355,6 +3603,15 @@ sub put_course_id_handler {
 #                            owner matches the supplied username and/or domain
 #                            will be returned. Pre-2.2.0 legacy entries from 
 #                            nohist_courseiddump will only contain usernames.
+#                 type     - optional parameter for selection 
+#                 regexp_ok - if true, allow the supplied institutional code
+#                            filter to behave as a regular expression.  
+#                 rtn_as_hash - whether to return the information available for
+#                            each matched item as a frozen hash of all 
+#                            key, value pairs in the item's hash, or as a 
+#                            colon-separated list of (in order) description,
+#                            institutional code, and course owner.
+#    
 #     $client  - The socket open on the client.
 # Returns:
 #    1     - Continue processing.
@@ -3362,11 +3619,11 @@ sub put_course_id_handler {
 #   a reply is written to $client.
 sub dump_course_id_handler {
     my ($cmd, $tail, $client) = @_;
-
     my $userinput = "$cmd:$tail";
 
     my ($udom,$since,$description,$instcodefilter,$ownerfilter,$coursefilter,
-        $typefilter,$regexp_ok) =split(/:/,$tail);
+        $typefilter,$regexp_ok,$rtn_as_hash,$selfenrollonly) =split(/:/,$tail);
+    my $now = time;
     if (defined($description)) {
 	$description=&unescape($description);
     } else {
@@ -3406,62 +3663,103 @@ sub dump_course_id_handler {
     if (defined($regexp_ok)) {
         $regexp_ok=&unescape($regexp_ok);
     }
-
-    unless (defined($since)) { $since=0; }
+    my $unpack = 1;
+    if ($description eq '.' && $instcodefilter eq '.' && $coursefilter eq '.' && 
+        $typefilter eq '.') {
+        $unpack = 0;
+    }
+    if (!defined($since)) { $since=0; }
     my $qresult='';
     my $hashref = &tie_domain_hash($udom, "nohist_courseids", &GDBM_WRCREAT());
     if ($hashref) {
 	while (my ($key,$value) = each(%$hashref)) {
-	    my ($descr,$lasttime,$inst_code,$owner,$type);
-            my @courseitems = split(/:/,$value);
-            $lasttime = pop(@courseitems);
-	    ($descr,$inst_code,$owner,$type)=@courseitems;
-	    if ($lasttime<$since) { next; }
+            my ($unesc_key,$lasttime_key,$lasttime,$is_hash,%val,
+                %unesc_val,$selfenroll_end,$selfenroll_types);
+            $unesc_key = &unescape($key);
+            if ($unesc_key =~ /^lasttime:/) {
+                next;
+            } else {
+                $lasttime_key = &escape('lasttime:'.$unesc_key);
+            }
+            if ($hashref->{$lasttime_key} ne '') {
+                $lasttime = $hashref->{$lasttime_key};
+                next if ($lasttime<$since);
+            }
+            my $items = &Apache::lonnet::thaw_unescape($value);
+            if (ref($items) eq 'HASH') {
+                $is_hash =  1;
+                if ($unpack || !$rtn_as_hash) {
+                    $unesc_val{'descr'} = $items->{'description'};
+                    $unesc_val{'inst_code'} = $items->{'inst_code'};
+                    $unesc_val{'owner'} = $items->{'owner'};
+                    $unesc_val{'type'} = $items->{'type'};
+                    $selfenroll_types = $items->{'selfenroll_types'};
+                    $selfenroll_end = $items->{'selfenroll_end_date'};
+                    if ($selfenrollonly) {
+                        next if (!$selfenroll_types);
+                        if (($selfenroll_end > 0) && ($selfenroll_end <= $now)) {
+                            next;
+                        }
+                    }
+                }
+            } else {
+                $is_hash =  0;
+                my @courseitems = split(/:/,$value);
+                $lasttime = pop(@courseitems);
+                next if ($lasttime<$since);
+	        ($val{'descr'},$val{'inst_code'},$val{'owner'},$val{'type'}) = @courseitems;
+            }
             my $match = 1;
-	    unless ($description eq '.') {
-		my $unescapeDescr = &unescape($descr);
-		unless (eval('$unescapeDescr=~/\Q$description\E/i')) {
+	    if ($description ne '.') {
+                if (!$is_hash) {
+                    $unesc_val{'descr'} = &unescape($val{'descr'});
+                }
+                if (eval{$unesc_val{'descr'} !~ /\Q$description\E/i}) {
                     $match = 0;
-		}
+                }
             }
-            unless ($instcodefilter eq '.' || !defined($instcodefilter)) {
-                my $unescapeInstcode = &unescape($inst_code);
+            if ($instcodefilter ne '.') {
+                if (!$is_hash) {
+                    $unesc_val{'inst_code'} = &unescape($val{'inst_code'});
+                }
                 if ($regexp_ok) {
-                    unless (eval('$unescapeInstcode=~/$instcodefilter/')) {
+                    if (eval{$unesc_val{'inst_code'} !~ /$instcodefilter/}) {
                         $match = 0;
                     }
                 } else {
-                    unless (eval('$unescapeInstcode=~/\Q$instcodefilter\E/i')) {
+                    if (eval{$unesc_val{'inst_code'} !~ /\Q$instcodefilter\E/i}) {
                         $match = 0;
                     }
                 }
 	    }
-            unless ($ownerfilter eq '.' || !defined($ownerfilter)) {
-                my $unescapeOwner = &unescape($owner);
+            if ($ownerfilter ne '.') {
+                if (!$is_hash) {
+                    $unesc_val{'owner'} = &unescape($val{'owner'});
+                }
                 if (($ownerunamefilter ne '') && ($ownerdomfilter ne '')) {
-                    if ($unescapeOwner =~ /:/) {
-                        if (eval('$unescapeOwner !~ 
-                             /\Q$ownerunamefilter\E:\Q$ownerdomfilter\E$/i')) {
+                    if ($unesc_val{'owner'} =~ /:/) {
+                        if (eval{$unesc_val{'owner'} !~ 
+                             /\Q$ownerunamefilter\E:\Q$ownerdomfilter\E$/i}) {
                             $match = 0;
                         } 
                     } else {
-                        if (eval('$unescapeOwner!~/\Q$ownerunamefilter\E/i')) {
+                        if (eval{$unesc_val{'owner'} !~ /\Q$ownerunamefilter\E/i}) {
                             $match = 0;
                         }
                     }
                 } elsif ($ownerunamefilter ne '') {
-                    if ($unescapeOwner =~ /:/) {
-                        if (eval('$unescapeOwner!~/\Q$ownerunamefilter\E:[^:]+$/i')) {
+                    if ($unesc_val{'owner'} =~ /:/) {
+                        if (eval{$unesc_val{'owner'} !~ /\Q$ownerunamefilter\E:[^:]+$/i}) {
                              $match = 0;
                         }
                     } else {
-                        if (eval('$unescapeOwner!~/\Q$ownerunamefilter\E/i')) {
+                        if (eval{$unesc_val{'owner'} !~ /\Q$ownerunamefilter\E/i}) {
                             $match = 0;
                         }
                     }
                 } elsif ($ownerdomfilter ne '') {
-                    if ($unescapeOwner =~ /:/) {
-                        if (eval('$unescapeOwner!~/^[^:]+:\Q$ownerdomfilter\E/')) {
+                    if ($unesc_val{'owner'} =~ /:/) {
+                        if (eval{$unesc_val{'owner'} !~ /^[^:]+:\Q$ownerdomfilter\E/}) {
                              $match = 0;
                         }
                     } else {
@@ -3471,31 +3769,53 @@ sub dump_course_id_handler {
                     }
                 }
             }
-            unless ($coursefilter eq '.' || !defined($coursefilter)) {
-                my $unescapeCourse = &unescape($key);
-                unless (eval('$unescapeCourse=~/^$udom(_)\Q$coursefilter\E$/')) {
+            if ($coursefilter ne '.') {
+                if (eval{$unesc_key !~ /^$udom(_)\Q$coursefilter\E$/}) {
                     $match = 0;
                 }
             }
-            unless ($typefilter eq '.' || !defined($typefilter)) {
-                my $unescapeType = &unescape($type);
-                if ($type eq '') {
+            if ($typefilter ne '.') {
+                if (!$is_hash) {
+                    $unesc_val{'type'} = &unescape($val{'type'});
+                }
+                if ($unesc_val{'type'} eq '') {
                     if ($typefilter ne 'Course') {
                         $match = 0;
                     }
-                } else { 
-                    unless (eval('$unescapeType=~/^\Q$typefilter\E$/')) {
+                } else {
+                    if (eval{$unesc_val{'type'} !~ /^\Q$typefilter\E$/}) {
                         $match = 0;
                     }
                 }
             }
             if ($match == 1) {
-                $qresult.=$key.'='.$descr.':'.$inst_code.':'.$owner.'&';
+                if ($rtn_as_hash) {
+                    if ($is_hash) {
+                        $qresult.=$key.'='.$value.'&';
+                    } else {
+                        my %rtnhash = ( 'description' => &unescape($val{'descr'}),
+                                        'inst_code' => &unescape($val{'inst_code'}),
+                                        'owner'     => &unescape($val{'owner'}),
+                                        'type'      => &unescape($val{'type'}),
+                                      );
+                        my $items = &Apache::lonnet::freeze_escape(\%rtnhash);
+                        $qresult.=$key.'='.$items.'&';
+                    }
+                } else {
+                    if ($is_hash) {
+                        $qresult .= $key.'='.&escape($unesc_val{'descr'}).':'.
+                                    &escape($unesc_val{'inst_code'}).':'.
+                                    &escape($unesc_val{'owner'}).'&';
+                    } else {
+                        $qresult .= $key.'='.$val{'descr'}.':'.$val{'inst_code'}.
+                                    ':'.$val{'owner'}.'&';
+                    }
+                }
             }
 	}
 	if (&untie_domain_hash($hashref)) {
 	    chop($qresult);
-	    &Reply($client, "$qresult\n", $userinput);
+	    &Reply($client, \$qresult, $userinput);
 	} else {
 	    &Failure($client, "error: ".($!+0)." untie(GDBM) Failed ".
 		    "while attempting courseiddump\n", $userinput);
@@ -3504,8 +3824,6 @@ sub dump_course_id_handler {
 	&Failure($client, "error: ".($!+0)." tie(GDBM) Failed ".
 		"while attempting courseiddump\n", $userinput);
     }
-
-
     return 1;
 }
 &register_handler("courseiddump", \&dump_course_id_handler, 0, 1, 0);
@@ -3588,7 +3906,7 @@ sub get_domain_handler {
         }
         if (&untie_domain_hash($hashref)) {
             $qresult=~s/\&$//;
-            &Reply($client, "$qresult\n", $userinput);
+            &Reply($client, \$qresult, $userinput);
         } else {
             &Failure( $client, "error: ".($!+0)." untie(GDBM) Failed ".
                       "while attempting getdom\n",$userinput);
@@ -3686,7 +4004,7 @@ sub get_id_handler {
 	}
 	if (&untie_domain_hash($hashref)) {
 	    $qresult=~s/\&$//;
-	    &Reply($client, "$qresult\n", $userinput);
+	    &Reply($client, \$qresult, $userinput);
 	} else {
 	    &Failure( $client, "error: ".($!+0)." untie(GDBM) Failed ".
 		      "while attempting idget\n",$userinput);
@@ -3810,7 +4128,7 @@ sub dump_dcmail_handler {
         }
         if (&untie_domain_hash($hashref)) {
             chop($qresult);
-            &Reply($client, "$qresult\n", $userinput);
+            &Reply($client, \$qresult, $userinput);
         } else {
             &Failure($client, "error: ".($!+0)." untie(GDBM) Failed ".
                     "while attempting dcmaildump\n", $userinput);
@@ -3928,7 +4246,7 @@ sub dump_domainroles_handler {
                 }
             }
             unless (@roles < 1) {
-                unless (grep/^$trole$/,@roles) {
+                unless (grep/^\Q$trole\E$/,@roles) {
                     $match = 0;
                 }
             }
@@ -3938,7 +4256,7 @@ sub dump_domainroles_handler {
         }
         if (&untie_domain_hash($hashref)) {
             chop($qresult);
-            &Reply($client, "$qresult\n", $userinput);
+            &Reply($client, \$qresult, $userinput);
         } else {
             &Failure($client, "error: ".($!+0)." untie(GDBM) Failed ".
                     "while attempting domrolesdump\n", $userinput);
@@ -3992,7 +4310,7 @@ sub tmp_put_handler {
     if ($store=IO::File->new(">$execdir/tmp/$id.tmp")) {
 	print $store $record;
 	close $store;
-	&Reply($client, "$id\n", $userinput);
+	&Reply($client, \$id, $userinput);
     } else {
 	&Failure( $client, "error: ".($!+0)."IO::File->new Failed ".
 		  "while attempting tmpput\n", $userinput);
@@ -4026,7 +4344,7 @@ sub tmp_get_handler {
     my $execdir=$perlvar{'lonDaemons'};
     if ($store=IO::File->new("$execdir/tmp/$id.tmp")) {
 	my $reply=<$store>;
-	&Reply( $client, "$reply\n", $userinput);
+	&Reply( $client, \$reply, $userinput);
 	close $store;
     } else {
 	&Failure( $client, "error: ".($!+0)."IO::File->new Failed ".
@@ -4210,7 +4528,7 @@ sub enrollment_enabled_handler {
     my ($cdom) = split(/:/, $tail, 2);   # Domain we're asking about.
 
     my $outcome  = &localenroll::run($cdom);
-    &Reply($client, "$outcome\n", $userinput);
+    &Reply($client, \$outcome, $userinput);
 
     return 1;
 }
@@ -4237,7 +4555,7 @@ sub get_sections_handler {
     my @secs = &localenroll::get_sections($coursecode,$cdom);
     my $seclist = &escape(join(':',@secs));
 
-    &Reply($client, "$seclist\n", $userinput);
+    &Reply($client, \$seclist, $userinput);
     
 
     return 1;
@@ -4266,7 +4584,7 @@ sub validate_course_owner_handler {
 
     $owner = &unescape($owner);
     my $outcome = &localenroll::new_course($inst_course_id,$owner,$cdom);
-    &Reply($client, "$outcome\n", $userinput);
+    &Reply($client, \$outcome, $userinput);
 
 
 
@@ -4297,7 +4615,7 @@ sub validate_course_section_handler {
     my ($inst_course_id, $cdom) = split(/:/, $tail);
 
     my $outcome=&localenroll::validate_courseID($inst_course_id,$cdom);
-    &Reply($client, "$outcome\n", $userinput);
+    &Reply($client, \$outcome, $userinput);
 
 
     return 1;
@@ -4324,14 +4642,14 @@ sub validate_course_section_handler {
 sub validate_class_access_handler {
     my ($cmd, $tail, $client) = @_;
     my $userinput = "$cmd:$tail";
-    my ($inst_class,$courseowner,$cdom) = split(/:/, $tail);
-    $courseowner = &unescape($courseowner);
+    my ($inst_class,$ownerlist,$cdom) = split(/:/, $tail);
+    my $owners = &unescape($ownerlist);
     my $outcome;
     eval {
 	local($SIG{__DIE__})='DEFAULT';
-	$outcome=&localenroll::check_section($inst_class,$courseowner,$cdom);
+	$outcome=&localenroll::check_section($inst_class,$owners,$cdom);
     };
-    &Reply($client,"$outcome\n", $userinput);
+    &Reply($client,\$outcome, $userinput);
 
     return 1;
 }
@@ -4452,10 +4770,10 @@ sub get_institutional_code_format_handle
 						    \%cat_titles,
 						    \%cat_order);
     if ($formatreply eq 'ok') {
-	my $codes_str = &hash2str(%codes);
-	my $codetitles_str = &array2str(@codetitles);
-	my $cat_titles_str = &hash2str(%cat_titles);
-	my $cat_order_str = &hash2str(%cat_order);
+	my $codes_str = &Apache::lonnet::hash2str(%codes);
+	my $codetitles_str = &Apache::lonnet::array2str(@codetitles);
+	my $cat_titles_str = &Apache::lonnet::hash2str(%cat_titles);
+	my $cat_order_str = &Apache::lonnet::hash2str(%cat_order);
 	&Reply($client,
 	       $codes_str.':'.$codetitles_str.':'.$cat_titles_str.':'
 	       .$cat_order_str."\n",
@@ -4492,7 +4810,7 @@ sub get_institutional_defaults_handler {
                 $result.=&escape($key).'='.&escape($value).'&';
             }
             $result .= 'code_order='.&escape(join('&',@code_order));
-            &Reply($client,$result."\n",$userinput);
+            &Reply($client,\$result,$userinput);
         } else {
             &Reply($client,"error\n", $userinput);
         }
@@ -4503,6 +4821,195 @@ sub get_institutional_defaults_handler {
 &register_handler("autoinstcodedefaults",
                   \&get_institutional_defaults_handler,0,1,0);
 
+sub get_institutional_user_rules {
+    my ($cmd, $tail, $client)   = @_;
+    my $userinput               = "$cmd:$tail";
+    my $dom = &unescape($tail);
+    my (%rules_hash,@rules_order);
+    my $outcome;
+    eval {
+        local($SIG{__DIE__})='DEFAULT';
+        $outcome = &localenroll::username_rules($dom,\%rules_hash,\@rules_order);
+    };
+    if (!$@) {
+        if ($outcome eq 'ok') {
+            my $result;
+            foreach my $key (keys(%rules_hash)) {
+                $result .= &escape($key).'='.&Apache::lonnet::freeze_escape($rules_hash{$key}).'&';
+            }
+            $result =~ s/\&$//;
+            $result .= ':';
+            if (@rules_order > 0) {
+                foreach my $item (@rules_order) {
+                    $result .= &escape($item).'&';
+                }
+            }
+            $result =~ s/\&$//;
+            &Reply($client,\$result,$userinput);
+        } else {
+            &Reply($client,"error\n", $userinput);
+        }
+    } else {
+        &Failure($client,"unknown_cmd\n",$userinput);
+    }
+}
+&register_handler("instuserrules",\&get_institutional_user_rules,0,1,0);
+
+sub get_institutional_id_rules {
+    my ($cmd, $tail, $client)   = @_;
+    my $userinput               = "$cmd:$tail";
+    my $dom = &unescape($tail);
+    my (%rules_hash,@rules_order);
+    my $outcome;
+    eval {
+        local($SIG{__DIE__})='DEFAULT';
+        $outcome = &localenroll::id_rules($dom,\%rules_hash,\@rules_order);
+    };
+    if (!$@) {
+        if ($outcome eq 'ok') {
+            my $result;
+            foreach my $key (keys(%rules_hash)) {
+                $result .= &escape($key).'='.&Apache::lonnet::freeze_escape($rules_hash{$key}).'&';
+            }
+            $result =~ s/\&$//;
+            $result .= ':';
+            if (@rules_order > 0) {
+                foreach my $item (@rules_order) {
+                    $result .= &escape($item).'&';
+                }
+            }
+            $result =~ s/\&$//;
+            &Reply($client,\$result,$userinput);
+        } else {
+            &Reply($client,"error\n", $userinput);
+        }
+    } else {
+        &Failure($client,"unknown_cmd\n",$userinput);
+    }
+}
+&register_handler("instidrules",\&get_institutional_id_rules,0,1,0);
+
+sub get_institutional_selfcreate_rules {
+    my ($cmd, $tail, $client)   = @_;
+    my $userinput               = "$cmd:$tail";
+    my $dom = &unescape($tail);
+    my (%rules_hash,@rules_order);
+    my $outcome;
+    eval {
+        local($SIG{__DIE__})='DEFAULT';
+        $outcome = &localenroll::selfcreate_rules($dom,\%rules_hash,\@rules_order);
+    };
+    if (!$@) {
+        if ($outcome eq 'ok') {
+            my $result;
+            foreach my $key (keys(%rules_hash)) {
+                $result .= &escape($key).'='.&Apache::lonnet::freeze_escape($rules_hash{$key}).'&';
+            }
+            $result =~ s/\&$//;
+            $result .= ':';
+            if (@rules_order > 0) {
+                foreach my $item (@rules_order) {
+                    $result .= &escape($item).'&';
+                }
+            }
+            $result =~ s/\&$//;
+            &Reply($client,\$result,$userinput);
+        } else {
+            &Reply($client,"error\n", $userinput);
+        }
+    } else {
+        &Failure($client,"unknown_cmd\n",$userinput);
+    }
+}
+&register_handler("instemailrules",\&get_institutional_selfcreate_rules,0,1,0);
+
+
+sub institutional_username_check {
+    my ($cmd, $tail, $client)   = @_;
+    my $userinput               = "$cmd:$tail";
+    my %rulecheck;
+    my $outcome;
+    my ($udom,$uname,@rules) = split(/:/,$tail);
+    $udom = &unescape($udom);
+    $uname = &unescape($uname);
+    @rules = map {&unescape($_);} (@rules);
+    eval {
+        local($SIG{__DIE__})='DEFAULT';
+        $outcome = &localenroll::username_check($udom,$uname,\@rules,\%rulecheck);
+    };
+    if (!$@) {
+        if ($outcome eq 'ok') {
+            my $result='';
+            foreach my $key (keys(%rulecheck)) {
+                $result.=&escape($key).'='.&Apache::lonnet::freeze_escape($rulecheck{$key}).'&';
+            }
+            &Reply($client,\$result,$userinput);
+        } else {
+            &Reply($client,"error\n", $userinput);
+        }
+    } else {
+        &Failure($client,"unknown_cmd\n",$userinput);
+    }
+}
+&register_handler("instrulecheck",\&institutional_username_check,0,1,0);
+
+sub institutional_id_check {
+    my ($cmd, $tail, $client)   = @_;
+    my $userinput               = "$cmd:$tail";
+    my %rulecheck;
+    my $outcome;
+    my ($udom,$id,@rules) = split(/:/,$tail);
+    $udom = &unescape($udom);
+    $id = &unescape($id);
+    @rules = map {&unescape($_);} (@rules);
+    eval {
+        local($SIG{__DIE__})='DEFAULT';
+        $outcome = &localenroll::id_check($udom,$id,\@rules,\%rulecheck);
+    };
+    if (!$@) {
+        if ($outcome eq 'ok') {
+            my $result='';
+            foreach my $key (keys(%rulecheck)) {
+                $result.=&escape($key).'='.&Apache::lonnet::freeze_escape($rulecheck{$key}).'&';
+            }
+            &Reply($client,\$result,$userinput);
+        } else {
+            &Reply($client,"error\n", $userinput);
+        }
+    } else {
+        &Failure($client,"unknown_cmd\n",$userinput);
+    }
+}
+&register_handler("instidrulecheck",\&institutional_id_check,0,1,0);
+
+sub institutional_selfcreate_check {
+    my ($cmd, $tail, $client)   = @_;
+    my $userinput               = "$cmd:$tail";
+    my %rulecheck;
+    my $outcome;
+    my ($udom,$email,@rules) = split(/:/,$tail);
+    $udom = &unescape($udom);
+    $email = &unescape($email);
+    @rules = map {&unescape($_);} (@rules);
+    eval {
+        local($SIG{__DIE__})='DEFAULT';
+        $outcome = &localenroll::selfcreate_check($udom,$email,\@rules,\%rulecheck);
+    };
+    if (!$@) {
+        if ($outcome eq 'ok') {
+            my $result='';
+            foreach my $key (keys(%rulecheck)) {
+                $result.=&escape($key).'='.&Apache::lonnet::freeze_escape($rulecheck{$key}).'&';
+            }
+            &Reply($client,\$result,$userinput);
+        } else {
+            &Reply($client,"error\n", $userinput);
+        }
+    } else {
+        &Failure($client,"unknown_cmd\n",$userinput);
+    }
+}
+&register_handler("instselfcreatecheck",\&institutional_selfcreate_check,0,1,0);
 
 # Get domain specific conditions for import of student photographs to a course
 #
@@ -4635,8 +5142,12 @@ sub inst_usertypes_handler {
     my ($cmd, $domain, $client) = @_;
     my $res;
     my $userinput = $cmd.":".$domain; # For logging purposes.
-    my (%typeshash,@order);  
-    if (&localenroll::inst_usertypes($domain,\%typeshash,\@order) eq 'ok') {
+    my (%typeshash,@order,$result);
+    eval {
+	local($SIG{__DIE__})='DEFAULT';
+	$result=&localenroll::inst_usertypes($domain,\%typeshash,\@order);
+    };
+    if ($result eq 'ok') {
         if (keys(%typeshash) > 0) {
             foreach my $key (keys(%typeshash)) {
                 $res.=&escape($key).'='.&escape($typeshash{$key}).'&';
@@ -4651,7 +5162,7 @@ sub inst_usertypes_handler {
         }
         $res=~s/\&$//;
     }
-    &Reply($client, "$res\n", $userinput);
+    &Reply($client, \$res, $userinput);
     return 1;
 }
 &register_handler("inst_usertypes", \&inst_usertypes_handler, 0, 1, 0);
@@ -4919,7 +5430,7 @@ sub catchexception {
     $SIG{__DIE__}='DEFAULT';
     &status("Catching exception");
     &logthis("<font color='red'>CRITICAL: "
-     ."ABNORMAL EXIT. Child $$ for server $thisserver died through "
+     ."ABNORMAL EXIT. Child $$ for server ".$perlvar{'lonHostID'}." died through "
      ."a crash with this error msg->[$error]</font>");
     &logthis('Famous last words: '.$status.' - '.$lastlog);
     if ($client) { print $client "error: $error\n"; }
@@ -5030,67 +5541,6 @@ sub HUPSMAN {                      # sig
 }
 
 #
-#    Kill off hashes that describe the host table prior to re-reading it.
-#    Hashes affected are:
-#       %hostid, %hostdom %hostip %hostdns.
-#
-sub KillHostHashes {
-    foreach my $key (keys %hostid) {
-	delete $hostid{$key};
-    }
-    foreach my $key (keys %hostdom) {
-	delete $hostdom{$key};
-    }
-    foreach my $key (keys %hostip) {
-	delete $hostip{$key};
-    }
-    foreach my $key (keys %hostdns) {
-	delete $hostdns{$key};
-    }
-}
-#
-#   Read in the host table from file and distribute it into the various hashes:
-#
-#    - %hostid  -  Indexed by IP, the loncapa hostname.
-#    - %hostdom -  Indexed by  loncapa hostname, the domain.
-#    - %hostip  -  Indexed by hostid, the Ip address of the host.
-sub ReadHostTable {
-
-    open (CONFIG,"$perlvar{'lonTabDir'}/hosts.tab") || die "Can't read host file";
-    my $myloncapaname = $perlvar{'lonHostID'};
-    Debug("My loncapa name is : $myloncapaname");
-    my %name_to_ip;
-    while (my $configline=<CONFIG>) {
-	if ($configline !~ /^\s*\#/ && $configline !~ /^\s*$/ ) {
-	    my ($id,$domain,$role,$name)=split(/:/,$configline);
-	    $name=~s/\s//g;
-	    my $ip;
-	    if (!exists($name_to_ip{$name})) {
-		$ip = gethostbyname($name);
-		if (!$ip || length($ip) ne 4) {
-		    &logthis("Skipping host $id name $name no IP found\n");
-		    next;
-		}
-		$ip=inet_ntoa($ip);
-		$name_to_ip{$name} = $ip;
-	    } else {
-		$ip = $name_to_ip{$name};
-	    }
-	    $hostid{$ip}=$id;         # LonCAPA name of host by IP.
-	    $hostdom{$id}=$domain;    # LonCAPA domain name of host. 
-	    $hostname{$id}=$name;     # LonCAPA name -> DNS name
-	    $hostip{$id}=$ip;         # IP address of host.
-	    $hostdns{$name} = $id;    # LonCAPA name of host by DNS.
-
-	    if ($id eq $perlvar{'lonHostID'}) { 
-		Debug("Found me in the host table: $name");
-		$thisserver=$name; 
-	    }
-	}
-    }
-    close(CONFIG);
-}
-#
 #  Reload the Apache daemon's state.
 #  This is done by invoking /home/httpd/perl/apachereload
 #  a setuid perl script that can be root for us to do this job.
@@ -5121,13 +5571,12 @@ sub UpdateHosts {
     #  either dropped or changed hosts.  Note that the re-read of the table
     #  will take care of new and changed hosts as connections come into being.
 
+    &Apache::lonnet::reset_hosts_info();
 
-    KillHostHashes;
-    ReadHostTable;
-
-    foreach my $child (keys %children) {
+    foreach my $child (keys(%children)) {
 	my $childip = $children{$child};
-	if(!$hostid{$childip}) {
+	if ($childip ne '127.0.0.1'
+	    && !defined(&Apache::lonnet::get_hosts_from_ip($childip))) {
 	    logthis('<font color="blue"> UpdateHosts killing child '
 		    ." $child for ip $childip </font>");
 	    kill('INT', $child);
@@ -5212,9 +5661,14 @@ sub Debug {
 #
 sub Reply {
     my ($fd, $reply, $request) = @_;
-    print $fd $reply;
-    Debug("Request was $request  Reply was $reply");
-
+    if (ref($reply)) {
+	print $fd $$reply;
+	print $fd "\n";
+	if ($DEBUG) { Debug("Request was $request  Reply was $$reply"); }
+    } else {
+	print $fd $reply;
+	if ($DEBUG) { Debug("Request was $request  Reply was $reply"); }
+    }
     $Transactions++;
 }
 
@@ -5287,84 +5741,6 @@ sub status {
     $0='lond: '.$what.' '.$local;
 }
 
-# ----------------------------------------------------------- Send USR1 to lonc
-
-sub reconlonc {
-    my $peerfile=shift;
-    &logthis("Trying to reconnect for $peerfile");
-    my $loncfile="$perlvar{'lonDaemons'}/logs/lonc.pid";
-    if (my $fh=IO::File->new("$loncfile")) {
-	my $loncpid=<$fh>;
-        chomp($loncpid);
-        if (kill 0 => $loncpid) {
-	    &logthis("lonc at pid $loncpid responding, sending USR1");
-            kill USR1 => $loncpid;
-        } else {
-	    &logthis(
-              "<font color='red'>CRITICAL: "
-             ."lonc at pid $loncpid not responding, giving up</font>");
-        }
-    } else {
-      &logthis('<font color="red">CRITICAL: lonc not running, giving up</font>');
-    }
-}
-
-sub create_connection {
-    my ($hostname,$lonid) = @_;
-    my $client=IO::Socket::UNIX->new(Peer    => $perlvar{'lonSockCreate'},
-				     Type    => SOCK_STREAM,
-				     Timeout => 10);
-    return 0 if (!$client);
-    print $client ("$hostname:$lonid\n");
-    my $result = <$client>;
-    chomp($result);
-    return 1 if ($result eq 'done');
-    return 0;
-}
-
-# -------------------------------------------------- Non-critical communication
-my $max_connection_retries = 10;
-sub subreply {
-    my ($cmd,$server)=@_;
-    my $peerfile="$perlvar{'lonSockDir'}/".$hostname{$server};
-    my $sclient;
-    for (my $retries = 0; $retries < $max_connection_retries; $retries++) {
-	$sclient=IO::Socket::UNIX->new(Peer    =>"$peerfile",
-				       Type    => SOCK_STREAM,
-				       Timeout => 10);
-	if($sclient) {
-	    last;		# Connected!
-	} else {
-	    &create_connection($hostname{$server},$server);
-	}
-        sleep(1);		# Try again later if failed connection.
-    }
-    print $sclient "sethost:$server:$cmd\n";
-    my $answer=<$sclient>;
-    chomp($answer);
-    if (!$answer) { $answer="con_lost"; }
-    return $answer;
-}
-
-sub reply {
-  my ($cmd,$server)=@_;
-  my $answer;
-  if ($server ne $currenthostid) { 
-    $answer=subreply($cmd,$server);
-    if ($answer eq 'con_lost') {
-	$answer=subreply("ping",$server);
-        if ($answer ne $server) {
-	    &logthis("sub reply: answer != server answer is $answer, server is $server");
-           &reconlonc("$perlvar{'lonSockDir'}/".$hostname{$server});
-        }
-        $answer=subreply($cmd,$server);
-    }
-  } else {
-    $answer='self_reply';
-  } 
-  return $answer;
-}
-
 # -------------------------------------------------------------- Talk to lonsql
 
 sub sql_reply {
@@ -5433,8 +5809,7 @@ $SIG{USR1} = \&checkchildren;
 $SIG{USR2} = \&UpdateHosts;
 
 #  Read the host hashes:
-
-ReadHostTable;
+&Apache::lonnet::load_hosts_tab();
 
 my $dist=`$perlvar{'lonDaemons'}/distprobe`;
 
@@ -5524,19 +5899,17 @@ sub make_new_child {
 # -----------------------------------------------------------------------------
 	# see if we know client and 'check' for spoof IP by ineffective challenge
 
-	ReadManagerTable;	# May also be a manager!!
-	
 	my $outsideip=$clientip;
 	if ($clientip eq '127.0.0.1') {
-	    $outsideip=$hostip{$perlvar{'lonHostID'}};
+	    $outsideip=&Apache::lonnet::get_host_ip($perlvar{'lonHostID'});
 	}
 
-	my $clientrec=($hostid{$outsideip}     ne undef);
+	my $clientrec=defined(&Apache::lonnet::get_hosts_from_ip($outsideip));
 	my $ismanager=($managers{$outsideip}    ne undef);
 	$clientname  = "[unknonwn]";
 	if($clientrec) {	# Establish client type.
 	    $ConnectionType = "client";
-	    $clientname = $hostid{$outsideip};
+	    $clientname = (&Apache::lonnet::get_hosts_from_ip($outsideip))[-1];
 	    if($ismanager) {
 		$ConnectionType = "both";
 	    }
@@ -5643,14 +6016,9 @@ sub make_new_child {
  
 	if ($clientok) {
 # ---------------- New known client connecting, could mean machine online again
-	    
-	    foreach my $id (keys(%hostip)) {
-		if ($hostip{$id} ne $clientip ||
-		    $hostip{$currenthostid} eq $clientip) {
-		    # no need to try to do recon's to myself
-		    next;
-		}
-		&reconlonc("$perlvar{'lonSockDir'}/".$hostname{$id});
+	    if (&Apache::lonnet::get_host_ip($currenthostid) ne $clientip 
+		&& $clientip ne '127.0.0.1') {
+		&Apache::lonnet::reconlonc($clientname);
 	    }
 	    &logthis("<font color='green'>Established connection: $clientname</font>");
 	    &status('Will listen to '.$clientname);
@@ -5853,8 +6221,7 @@ sub get_auth_type
 #     0        - The domain,user,password triplet is not a valid user.
 #
 sub validate_user {
-    my ($domain, $user, $password) = @_;
-
+    my ($domain, $user, $password, $checkdefauth) = @_;
 
     # Why negative ~pi you may well ask?  Well this function is about
     # authentication, and therefore very important to get right.
@@ -5877,8 +6244,21 @@ sub validate_user {
 
     my $null = pack("C",0);	# Used by kerberos auth types.
 
+    if ($howpwd eq 'nouser') {
+        if ($checkdefauth) {
+            my %domdefaults = &Apache::lonnet::get_domain_defaults($domain);
+            if ($domdefaults{'auth_def'} eq 'localauth') {
+                $howpwd = $domdefaults{'auth_def'};
+                $contentpwd = $domdefaults{'auth_arg_def'};
+            } elsif ((($domdefaults{'auth_def'} eq 'krb4') || 
+                      ($domdefaults{'auth_def'} eq 'krb5')) &&
+                     ($domdefaults{'auth_arg_def'} ne '')) {
+                $howpwd = $domdefaults{'auth_def'};
+                $contentpwd = $domdefaults{'auth_arg_def'}; 
+            }
+        }
+    } 
     if ($howpwd ne 'nouser') {
-
 	if($howpwd eq "internal") { # Encrypted is in local password file.
 	    $validated = (crypt($password, $contentpwd) eq $contentpwd);
 	}
@@ -5929,11 +6309,22 @@ sub validate_user {
 		my $credentials= &Authen::Krb5::cc_default();
 		$credentials->initialize(&Authen::Krb5::parse_name($user.'@'
                                                                  .$contentpwd));
-		my $krbreturn  = &Authen::Krb5::get_in_tkt_with_password($krbclient,
-									 $krbserver,
-									 $password,
-									 $credentials);
-		$validated = ($krbreturn == 1);
+                my $krbreturn;
+                if (exists(&Authen::Krb5::get_init_creds_password)) {
+                    $krbreturn = 
+                        &Authen::Krb5::get_init_creds_password($krbclient,$password,
+                                                               $krbservice);
+                    $validated = (ref($krbreturn) eq 'Authen::Krb5::Creds');
+                } else {
+		    $krbreturn  = 
+                        &Authen::Krb5::get_in_tkt_with_password($krbclient,$krbserver,
+		 						$password,$credentials);
+		    $validated = ($krbreturn == 1);
+                }
+		if (!$validated) {
+		    &logthis('krb5: '.$user.', '.$contentpwd.', '.
+			     &Authen::Krb5::error());
+		}
 	    } else {
 		$validated = 0;
 	    }
@@ -6179,7 +6570,7 @@ sub subscribe {
                 # the metadata
 		unless ($fname=~/\.meta$/) { &unsub("$fname.meta",$clientip); }
 		$fname=~s/\/home\/httpd\/html\/res/raw/;
-		$fname="http://$thisserver/".$fname;
+		$fname="http://".&Apache::lonnet::hostname($perlvar{'lonHostID'})."/".$fname;
 		$result="$fname\n";
 	    }
 	} else {
@@ -6222,7 +6613,7 @@ sub change_unix_password {
 
 sub make_passwd_file {
     my ($uname, $umode,$npass,$passfilename)=@_;
-    my $result="ok\n";
+    my $result="ok";
     if ($umode eq 'krb4' or $umode eq 'krb5') {
 	{
 	    my $pf = IO::File->new(">$passfilename");
@@ -6290,7 +6681,7 @@ sub make_passwd_file {
 		if($useraddok > 0) {
 		    my $error_text = &lcuseraddstrerror($useraddok);
 		    &logthis("Failed lcuseradd: $error_text");
-		    $result = "lcuseradd_failed:$error_text\n";
+		    $result = "lcuseradd_failed:$error_text";
 		}  else {
 		    my $pf = IO::File->new(">$passfilename");
 		    if($pf) {
@@ -6314,7 +6705,7 @@ sub make_passwd_file {
 	    }
 	}
     } else {
-	$result="auth_mode_error\n";
+	$result="auth_mode_error";
     }
     return $result;
 }
@@ -6333,9 +6724,10 @@ sub sethost {
     }
 
     if (!defined($hostid)) { $hostid=$perlvar{'lonHostID'}; }
-    if ($hostip{$perlvar{'lonHostID'}} eq $hostip{$hostid}) {
+    if (&Apache::lonnet::get_host_ip($perlvar{'lonHostID'}) 
+	eq &Apache::lonnet::get_host_ip($hostid)) {
 	$currenthostid  =$hostid;
-	$currentdomainid=$hostdom{$hostid};
+	$currentdomainid=&Apache::lonnet::host_domain($hostid);
 	&logthis("Setting hostid to $hostid, and domain to $currentdomainid");
     } else {
 	&logthis("Requested host id $hostid not an alias of ".
@@ -6351,96 +6743,6 @@ sub version {
     return "version:$VERSION";
 }
 
-#There is a copy of this in lonnet.pm
-sub userload {
-    my $numusers=0;
-    {
-	opendir(LONIDS,$perlvar{'lonIDsDir'});
-	my $filename;
-	my $curtime=time;
-	while ($filename=readdir(LONIDS)) {
-	    if ($filename eq '.' || $filename eq '..') {next;}
-	    my ($mtime)=(stat($perlvar{'lonIDsDir'}.'/'.$filename))[9];
-	    if ($curtime-$mtime < 1800) { $numusers++; }
-	}
-	closedir(LONIDS);
-    }
-    my $userloadpercent=0;
-    my $maxuserload=$perlvar{'lonUserLoadLim'};
-    if ($maxuserload) {
-	$userloadpercent=100*$numusers/$maxuserload;
-    }
-    $userloadpercent=sprintf("%.2f",$userloadpercent);
-    return $userloadpercent;
-}
-
-# Routines for serializing arrays and hashes (copies from lonnet)
-
-sub array2str {
-  my (@array) = @_;
-  my $result=&arrayref2str(\@array);
-  $result=~s/^__ARRAY_REF__//;
-  $result=~s/__END_ARRAY_REF__$//;
-  return $result;
-}
-                                                                                 
-sub arrayref2str {
-  my ($arrayref) = @_;
-  my $result='__ARRAY_REF__';
-  foreach my $elem (@$arrayref) {
-    if(ref($elem) eq 'ARRAY') {
-      $result.=&arrayref2str($elem).'&';
-    } elsif(ref($elem) eq 'HASH') {
-      $result.=&hashref2str($elem).'&';
-    } elsif(ref($elem)) {
-      #print("Got a ref of ".(ref($elem))." skipping.");
-    } else {
-      $result.=&escape($elem).'&';
-    }
-  }
-  $result=~s/\&$//;
-  $result .= '__END_ARRAY_REF__';
-  return $result;
-}
-                                                                                 
-sub hash2str {
-  my (%hash) = @_;
-  my $result=&hashref2str(\%hash);
-  $result=~s/^__HASH_REF__//;
-  $result=~s/__END_HASH_REF__$//;
-  return $result;
-}
-                                                                                 
-sub hashref2str {
-  my ($hashref)=@_;
-  my $result='__HASH_REF__';
-  foreach (sort(keys(%$hashref))) {
-    if (ref($_) eq 'ARRAY') {
-      $result.=&arrayref2str($_).'=';
-    } elsif (ref($_) eq 'HASH') {
-      $result.=&hashref2str($_).'=';
-    } elsif (ref($_)) {
-      $result.='=';
-      #print("Got a ref of ".(ref($_))." skipping.");
-    } else {
-        if ($_) {$result.=&escape($_).'=';} else { last; }
-    }
-
-    if(ref($hashref->{$_}) eq 'ARRAY') {
-      $result.=&arrayref2str($hashref->{$_}).'&';
-    } elsif(ref($hashref->{$_}) eq 'HASH') {
-      $result.=&hashref2str($hashref->{$_}).'&';
-    } elsif(ref($hashref->{$_})) {
-       $result.='&';
-      #print("Got a ref of ".(ref($hashref->{$_}))." skipping.");
-    } else {
-      $result.=&escape($hashref->{$_}).'&';
-    }
-  }
-  $result=~s/\&$//;
-  $result .= '__END_HASH_REF__';
-  return $result;
-}
 
 # ----------------------------------- POD (plain old documentation, CPAN style)