--- loncom/lonnet/perl/lonnet.pm	2006/11/21 20:58:06	1.806
+++ loncom/lonnet/perl/lonnet.pm	2007/06/25 18:12:24	1.894
@@ -1,7 +1,7 @@
 # The LearningOnline Network
 # TCP networking package
 #
-# $Id: lonnet.pm,v 1.806 2006/11/21 20:58:06 raeburn Exp $
+# $Id: lonnet.pm,v 1.894 2007/06/25 18:12:24 albertel Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -31,30 +31,26 @@ package Apache::lonnet;
 
 use strict;
 use LWP::UserAgent();
-use HTTP::Headers;
 use HTTP::Date;
 # use Date::Parse;
-use vars 
-qw(%perlvar %hostname %badServerCache %iphost %spareid %hostdom 
-   %libserv %pr %prp $memcache %packagetab 
-   %courselogs %accesshash %userrolehash %domainrolehash $processmarker $dumpcount 
-   %coursedombuf %coursenumbuf %coursehombuf %coursedescrbuf %courseinstcodebuf %courseownerbuf %coursetypebuf
-   %domaindescription %domain_auth_def %domain_auth_arg_def 
-   %domain_lang_def %domain_city %domain_longi %domain_lati %domain_primary
-   $tmpdir $_64bit %env);
+use vars qw(%perlvar %spareid %pr %prp $memcache %packagetab $tmpdir
+            $_64bit %env);
+
+my (%badServerCache, $memcache, %courselogs, %accesshash, %domainrolehash,
+    %userrolehash, $processmarker, $dumpcount, %coursedombuf,
+    %coursenumbuf, %coursehombuf, %coursedescrbuf, %courseinstcodebuf,
+    %courseownerbuf, %coursetypebuf);
 
 use IO::Socket;
 use GDBM_File;
 use HTML::LCParser;
-use HTML::Parser;
 use Fcntl qw(:flock);
-use Storable qw(lock_store lock_nstore lock_retrieve freeze thaw nfreeze);
+use Storable qw(thaw nfreeze);
 use Time::HiRes qw( gettimeofday tv_interval );
 use Cache::Memcached;
 use Digest::MD5;
 use Math::Random;
-use lib '/home/httpd/lib/perl';
-use LONCAPA;
+use LONCAPA qw(:DEFAULT :match);
 use LONCAPA::Configuration;
 
 my $readit;
@@ -147,10 +143,24 @@ sub logperm {
     return 1;
 }
 
+sub create_connection {
+    my ($hostname,$lonid) = @_;
+    my $client=IO::Socket::UNIX->new(Peer    => $perlvar{'lonSockCreate'},
+				     Type    => SOCK_STREAM,
+				     Timeout => 10);
+    return 0 if (!$client);
+    print $client (join(':',$hostname,$lonid,&machine_ids($hostname))."\n");
+    my $result = <$client>;
+    chomp($result);
+    return 1 if ($result eq 'done');
+    return 0;
+}
+
+
 # -------------------------------------------------- Non-critical communication
 sub subreply {
     my ($cmd,$server)=@_;
-    my $peerfile="$perlvar{'lonSockDir'}/".$hostname{$server};
+    my $peerfile="$perlvar{'lonSockDir'}/".&hostname($server);
     #
     #  With loncnew process trimming, there's a timing hole between lonc server
     #  process exit and the master server picking up the listen on the AF_UNIX
@@ -171,10 +181,12 @@ sub subreply {
 	$client=IO::Socket::UNIX->new(Peer    =>"$peerfile",
 				      Type    => SOCK_STREAM,
 				      Timeout => 10);
-	if($client) {
+	if ($client) {
 	    last;		# Connected!
+	} else {
+	    &create_connection(&hostname($server),$server);
 	}
-	sleep(1);		# Try again later if failed connection.
+        sleep(1);		# Try again later if failed connection.
     }
     my $answer;
     if ($client) {
@@ -190,7 +202,7 @@ sub subreply {
 
 sub reply {
     my ($cmd,$server)=@_;
-    unless (defined($hostname{$server})) { return 'no_such_host'; }
+    unless (defined(&hostname($server))) { return 'no_such_host'; }
     my $answer=subreply($cmd,$server);
     if (($answer=~/^refused/) || ($answer=~/^rejected/)) {
        &logthis("<font color=\"blue\">WARNING:".
@@ -202,8 +214,25 @@ sub reply {
 # ----------------------------------------------------------- Send USR1 to lonc
 
 sub reconlonc {
-    my $peerfile=shift;
-    &logthis("Trying to reconnect for $peerfile");
+    my ($lonid) = @_;
+    my $hostname = &hostname($lonid);
+    if ($lonid) {
+	my $peerfile="$perlvar{'lonSockDir'}/$hostname";
+	if ($hostname && -e $peerfile) {
+	    &logthis("Trying to reconnect lonc for $lonid ($hostname)");
+	    my $client=IO::Socket::UNIX->new(Peer    => $peerfile,
+					     Type    => SOCK_STREAM,
+					     Timeout => 10);
+	    if ($client) {
+		print $client ("reset_retries\n");
+		my $answer=<$client>;
+		#reset just this one.
+	    }
+	}
+	return;
+    }
+
+    &logthis("Trying to reconnect lonc");
     my $loncfile="$perlvar{'lonDaemons'}/logs/lonc.pid";
     if (open(my $fh,"<$loncfile")) {
 	my $loncpid=<$fh>;
@@ -212,19 +241,13 @@ sub reconlonc {
 	    &logthis("lonc at pid $loncpid responding, sending USR1");
             kill USR1 => $loncpid;
             sleep 1;
-            if (-e "$peerfile") { return; }
-            &logthis("$peerfile still not there, give it another try");
-            sleep 5;
-            if (-e "$peerfile") { return; }
-            &logthis(
-  "<font color=\"blue\">WARNING: $peerfile still not there, giving up</font>");
-        } else {
+         } else {
 	    &logthis(
                "<font color=\"blue\">WARNING:".
                " lonc at pid $loncpid not responding, giving up</font>");
         }
     } else {
-     &logthis('<font color="blue">WARNING: lonc not running, giving up</font>');
+	&logthis('<font color="blue">WARNING: lonc not running, giving up</font>');
     }
 }
 
@@ -232,7 +255,7 @@ sub reconlonc {
 
 sub critical {
     my ($cmd,$server)=@_;
-    unless ($hostname{$server}) {
+    unless (&hostname($server)) {
         &logthis("<font color=\"blue\">WARNING:".
                " Critical message to unknown server ($server)</font>");
         return 'no_such_host';
@@ -368,6 +391,26 @@ sub transfer_profile_to_env {
     }
 }
 
+sub timed_flock {
+    my ($file,$lock_type) = @_;
+    my $failed=0;
+    eval {
+	local $SIG{__DIE__}='DEFAULT';
+	local $SIG{ALRM}=sub {
+	    $failed=1;
+	    die("failed lock");
+	};
+	alarm(13);
+	flock($file,$lock_type);
+	alarm(0);
+    };
+    if ($failed) {
+	return undef;
+    } else {
+	return 1;
+    }
+}
+
 # ---------------------------------------------------------- Append Environment
 
 sub appenv {
@@ -382,8 +425,11 @@ sub appenv {
             $env{$key}=$newenv{$key};
         }
     }
-    if (tie(my %disk_env,'GDBM_File',$env{'user.environment'},&GDBM_WRITER(),
-	    0640)) {
+    open(my $env_file,$env{'user.environment'});
+    if (&timed_flock($env_file,LOCK_EX)
+	&&
+	tie(my %disk_env,'GDBM_File',$env{'user.environment'},
+	    (&GDBM_WRITER()|&GDBM_NOLOCK()),0640)) {
 	while (my ($key,$value) = each(%newenv)) {
 	    $disk_env{$key} = $value;
 	}
@@ -400,8 +446,11 @@ sub delenv {
                 "Attempt to delete from environment ".$delthis);
         return 'error';
     }
-    if (tie(my %disk_env,'GDBM_File',$env{'user.environment'},&GDBM_WRITER(),
-	    0640)) {
+    open(my $env_file,$env{'user.environment'});
+    if (&timed_flock($env_file,LOCK_EX)
+	&&
+	tie(my %disk_env,'GDBM_File',$env{'user.environment'},
+	    (&GDBM_WRITER()|&GDBM_NOLOCK()),0640)) {
 	foreach my $key (keys(%disk_env)) {
 	    if ($key=~/^$delthis/) { 
                 delete($env{$key});
@@ -499,7 +548,7 @@ sub spareserver {
     }
 
     if (!$want_server_name) {
-	$spare_server="http://$hostname{$spare_server}";
+	$spare_server="http://".&hostname($spare_server);
     }
     return $spare_server;
 }
@@ -588,11 +637,17 @@ sub queryauthenticate {
 
 sub authenticate {
     my ($uname,$upass,$udom)=@_;
-    $upass=escape($upass);
-    $uname=~s/\W//g;
-    my $uhome=&homeserver($uname,$udom);
-    if (!$uhome) {
-	&logthis("User $uname at $udom is unknown in authenticate");
+    $upass=&escape($upass);
+    $uname= &LONCAPA::clean_username($uname);
+    my $uhome=&homeserver($uname,$udom,1);
+    if ((!$uhome) || ($uhome eq 'no_host')) {
+# Maybe the machine was offline and only re-appeared again recently?
+        &reconlonc();
+# One more
+	my $uhome=&homeserver($uname,$udom,1);
+	if ((!$uhome) || ($uhome eq 'no_host')) {
+	    &logthis("User $uname at $udom is unknown in authenticate");
+	}
 	return 'no_host';
     }
     my $answer=reply("encrypt:auth:$udom:$uname:$upass",$uhome);
@@ -616,18 +671,19 @@ sub homeserver {
     my $index="$uname:$udom";
 
     if (exists($homecache{$index})) { return $homecache{$index}; }
-    my $tryserver;
-    foreach $tryserver (keys %libserv) {
+
+    my %servers = &get_servers($udom,'library');
+    foreach my $tryserver (keys(%servers)) {
         next if ($ignoreBadCache ne 'true' && 
 		 exists($badServerCache{$tryserver}));
-	if ($hostdom{$tryserver} eq $udom) {
-           my $answer=reply("home:$udom:$uname",$tryserver);
-           if ($answer eq 'found') { 
-	       return $homecache{$index}=$tryserver;
-           } elsif ($answer eq 'no_host') {
-	       $badServerCache{$tryserver}=1;
-           }
-       }
+
+	my $answer=reply("home:$udom:$uname",$tryserver);
+	if ($answer eq 'found') {
+	    delete($badServerCache{$tryserver}); 
+	    return $homecache{$index}=$tryserver;
+	} elsif ($answer eq 'no_host') {
+	    $badServerCache{$tryserver}=1;
+	}
     }    
     return 'no_host';
 }
@@ -638,24 +694,22 @@ sub idget {
     my ($udom,@ids)=@_;
     my %returnhash=();
     
-    my $tryserver;
-    foreach $tryserver (keys %libserv) {
-       if ($hostdom{$tryserver} eq $udom) {
-	  my $idlist=join('&',@ids);
-          $idlist=~tr/A-Z/a-z/; 
-	  my $reply=&reply("idget:$udom:".$idlist,$tryserver);
-          my @answer=();
-          if (($reply ne 'con_lost') && ($reply!~/^error\:/)) {
-	      @answer=split(/\&/,$reply);
-          }                    ;
-          my $i;
-          for ($i=0;$i<=$#ids;$i++) {
-              if ($answer[$i]) {
-		  $returnhash{$ids[$i]}=$answer[$i];
-              } 
-          }
-       }
-    }    
+    my %servers = &get_servers($udom,'library');
+    foreach my $tryserver (keys(%servers)) {
+	my $idlist=join('&',@ids);
+	$idlist=~tr/A-Z/a-z/; 
+	my $reply=&reply("idget:$udom:".$idlist,$tryserver);
+	my @answer=();
+	if (($reply ne 'con_lost') && ($reply!~/^error\:/)) {
+	    @answer=split(/\&/,$reply);
+	}                    ;
+	my $i;
+	for ($i=0;$i<=$#ids;$i++) {
+	    if ($answer[$i]) {
+		$returnhash{$ids[$i]}=$answer[$i];
+	    } 
+	}
+    } 
     return %returnhash;
 }
 
@@ -697,21 +751,36 @@ sub idput {
 # ------------------------------------------- get items from domain db files   
 
 sub get_dom {
-    my ($namespace,$storearr,$udom)=@_;
+    my ($namespace,$storearr,$udom,$uhome)=@_;
     my $items='';
     foreach my $item (@$storearr) {
         $items.=&escape($item).'&';
     }
     $items=~s/\&$//;
-    if (!$udom) { $udom=$env{'user.domain'}; }
-    if (exists($domain_primary{$udom})) {
-        my $uhome=$domain_primary{$udom};
+    if (!$udom) {
+        $udom=$env{'user.domain'};
+        if (defined(&domain($udom,'primary'))) {
+            $uhome=&domain($udom,'primary');
+        } else {
+            undef($uhome);
+        }
+    } else {
+        if (!$uhome) {
+            if (defined(&domain($udom,'primary'))) {
+                $uhome=&domain($udom,'primary');
+            }
+        }
+    }
+    if ($udom && $uhome && ($uhome ne 'no_host')) {
         my $rep=&reply("getdom:$udom:$namespace:$items",$uhome);
+        my %returnhash;
+        if ($rep eq '' || $rep =~ /^error: 2 /) {
+            return %returnhash;
+        }
         my @pairs=split(/\&/,$rep);
         if ( $#pairs==0 && $pairs[0] =~ /^(con_lost|error|no_such_host)/i) {
             return @pairs;
         }
-        my %returnhash=();
         my $i=0;
         foreach my $item (@$storearr) {
             $returnhash{$item}=&thaw_unescape($pairs[$i]);
@@ -719,17 +788,29 @@ sub get_dom {
         }
         return %returnhash;
     } else {
-        &logthis("get_dom failed - no primary domain server for $udom");
+        &logthis("get_dom failed - no homeserver and/or domain ($udom) ($uhome)");
     }
 }
 
 # -------------------------------------------- put items in domain db files 
 
 sub put_dom {
-    my ($namespace,$storehash,$udom)=@_;
-    if (!$udom) { $udom=$env{'user.domain'}; }
-    if (exists($domain_primary{$udom})) {
-        my $uhome=$domain_primary{$udom};
+    my ($namespace,$storehash,$udom,$uhome)=@_;
+    if (!$udom) {
+        $udom=$env{'user.domain'};
+        if (defined(&domain($udom,'primary'))) {
+            $uhome=&domain($udom,'primary');
+        } else {
+            undef($uhome);
+        }
+    } else {
+        if (!$uhome) {
+            if (defined(&domain($udom,'primary'))) {
+                $uhome=&domain($udom,'primary');
+            }
+        }
+    } 
+    if ($udom && $uhome && ($uhome ne 'no_host')) {
         my $items='';
         foreach my $item (keys(%$storehash)) {
             $items.=&escape($item).'='.&freeze_escape($$storehash{$item}).'&';
@@ -737,8 +818,42 @@ sub put_dom {
         $items=~s/\&$//;
         return &reply("putdom:$udom:$namespace:$items",$uhome);
     } else {
-        &logthis("put_dom failed - no primary domain server for $udom");
+        &logthis("put_dom failed - no homeserver and/or domain");
+    }
+}
+
+sub retrieve_inst_usertypes {
+    my ($udom) = @_;
+    my (%returnhash,@order);
+    if (defined(&domain($udom,'primary'))) {
+        my $uhome=&domain($udom,'primary');
+        my $rep=&reply("inst_usertypes:$udom",$uhome);
+        my ($hashitems,$orderitems) = split(/:/,$rep); 
+        my @pairs=split(/\&/,$hashitems);
+        foreach my $item (@pairs) {
+            my ($key,$value)=split(/=/,$item,2);
+            $key = &unescape($key);
+            next if ($key =~ /^error: 2 /);
+            $returnhash{$key}=&thaw_unescape($value);
+        }
+        my @esc_order = split(/\&/,$orderitems);
+        foreach my $item (@esc_order) {
+            push(@order,&unescape($item));
+        }
+    } else {
+        &logthis("get_dom failed - no primary domain server for $udom");
+    }
+    return (\%returnhash,\@order);
+}
+
+sub is_domainimage {
+    my ($url) = @_;
+    if ($url=~m-^/+res/+($match_domain)/+\1\-domainconfig/+(img|logo|domlogo)/+-) {
+        if (&domain($1) ne '') {
+            return '1';
+        }
     }
+    return;
 }
 
 # --------------------------------------------------- Assign a key to a student
@@ -874,17 +989,32 @@ sub validate_access_key {
 # ------------------------------------- Find the section of student in a course
 sub devalidate_getsection_cache {
     my ($udom,$unam,$courseid)=@_;
-    $courseid=~s/\_/\//g;
-    $courseid=~s/^(\w)/\/$1/;
     my $hashid="$udom:$unam:$courseid";
     &devalidate_cache_new('getsection',$hashid);
 }
 
+sub courseid_to_courseurl {
+    my ($courseid) = @_;
+    #already url style courseid
+    return $courseid if ($courseid =~ m{^/});
+
+    if (exists($env{'course.'.$courseid.'.num'})) {
+	my $cnum = $env{'course.'.$courseid.'.num'};
+	my $cdom = $env{'course.'.$courseid.'.domain'};
+	return "/$cdom/$cnum";
+    }
+
+    my %courseinfo=&Apache::lonnet::coursedescription($courseid);
+    if (exists($courseinfo{'num'})) {
+	return "/$courseinfo{'domain'}/$courseinfo{'num'}";
+    }
+
+    return undef;
+}
+
 sub getsection {
     my ($udom,$unam,$courseid)=@_;
     my $cachetime=1800;
-    $courseid=~s/\_/\//g;
-    $courseid=~s/^(\w)/\/$1/;
 
     my $hashid="$udom:$unam:$courseid";
     my ($result,$cached)=&is_cached_new('getsection',$hashid);
@@ -905,14 +1035,13 @@ sub getsection {
     # If there is more than one expired role, choose the one which ended last.
     # If there is a role which has expired, return it.
     #
-    foreach my $line (split(/\&/,&reply('dump:'.$udom.':'.$unam.':roles',
-					&homeserver($unam,$udom)))) {
-        my ($key,$value)=split(/\=/,$line,2);
-        $key=&unescape($key);
+    $courseid = &courseid_to_courseurl($courseid);
+    my %roleshash = &dump('roles',$udom,$unam,$courseid);
+    foreach my $key (keys(%roleshash)) {
         next if ($key !~/^\Q$courseid\E(?:\/)*(\w+)*\_st$/);
         my $section=$1;
         if ($key eq $courseid.'_st') { $section=''; }
-        my ($dummy,$end,$start)=split(/\_/,&unescape($value));
+        my ($dummy,$end,$start)=split(/\_/,&unescape($roleshash{$key}));
         my $now=time;
         if (defined($end) && $end && ($now > $end)) {
             $Expired{$end}=$section;
@@ -951,10 +1080,19 @@ my %remembered;
 my %accessed;
 my $kicks=0;
 my $hits=0;
+sub make_key {
+    my ($name,$id) = @_;
+    if (length($id) > 65 
+	&& length(&escape($id)) > 200) {
+	$id=length($id).':'.&Digest::MD5::md5_hex($id);
+    }
+    return &escape($name.':'.$id);
+}
+
 sub devalidate_cache_new {
     my ($name,$id,$debug) = @_;
     if ($debug) { &Apache::lonnet::logthis("deleting $name:$id"); }
-    $id=&escape($name.':'.$id);
+    $id=&make_key($name,$id);
     $memcache->delete($id);
     delete($remembered{$id});
     delete($accessed{$id});
@@ -962,7 +1100,7 @@ sub devalidate_cache_new {
 
 sub is_cached_new {
     my ($name,$id,$debug) = @_;
-    $id=&escape($name.':'.$id);
+    $id=&make_key($name,$id);
     if (exists($remembered{$id})) {
 	if ($debug) { &Apache::lonnet::logthis("Earyl return $id of $remembered{$id} "); }
 	$accessed{$id}=[&gettimeofday()];
@@ -985,7 +1123,7 @@ sub is_cached_new {
 
 sub do_cache_new {
     my ($name,$id,$value,$time,$debug) = @_;
-    $id=&escape($name.':'.$id);
+    $id=&make_key($name,$id);
     my $setvalue=$value;
     if (!defined($setvalue)) {
 	$setvalue='__undef__';
@@ -994,7 +1132,9 @@ sub do_cache_new {
 	$time=600;
     }
     if ($debug) { &Apache::lonnet::logthis("Setting $id to $value"); }
-    $memcache->set($id,$setvalue,$time);
+    if (!($memcache->set($id,$setvalue,$time))) {
+	&logthis("caching of id -> $id  failed");
+    }
     # need to make a copy of $value
     #&make_room($id,$value,$debug);
     return $value;
@@ -1163,6 +1303,7 @@ sub repcopy {
     }
     $filename=~s/[\n\r]//g;
     my $transname="$filename.in.transfer";
+# FIXME: this should flock
     if ((-e $filename) || (-e $transname)) { return 'ok'; }
     my $remoteurl=subscribe($filename);
     if ($remoteurl =~ /^con_lost by/) {
@@ -1411,15 +1552,17 @@ sub store_edited_file {
 }
 
 sub clean_filename {
-    my ($fname)=@_;
+    my ($fname,$args)=@_;
 # Replace Windows backslashes by forward slashes
     $fname=~s/\\/\//g;
-# Get rid of everything but the actual filename
-    $fname=~s/^.*\/([^\/]+)$/$1/;
+    if (!$args->{'keep_path'}) {
+        # Get rid of everything but the actual filename
+	$fname=~s/^.*\/([^\/]+)$/$1/;
+    }
 # Replace spaces by underscores
     $fname=~s/\s+/\_/g;
 # Replace all other weird characters by nothing
-    $fname=~s/[^\w\.\-]//g;
+    $fname=~s{[^/\w\.\-]}{}g;
 # Replace all .\d. sequences with _\d. so they no longer look like version
 # numbers
     $fname=~s/\.(\d+)(?=\.)/_$1/g;
@@ -1432,14 +1575,21 @@ sub clean_filename {
 #        $coursedoc - if true up to the current course
 #                     if false
 #        $subdir - directory in userfile to store the file into
-#        $parser, $allfiles, $codebase - unknown
-#
+#        $parser - instruction to parse file for objects ($parser = parse)    
+#        $allfiles - reference to hash for embedded objects
+#        $codebase - reference to hash for codebase of java objects
+#        $desuname - username for permanent storage of uploaded file
+#        $dsetudom - domain for permanaent storage of uploaded file
+#        $thumbwidth - width (pixels) of thumbnail to make for uploaded image 
+#        $thumbheight - height (pixels) of thumbnail to make for uploaded image
+# 
 # output: url of file in userspace, or error: <message> 
 #             or /adm/notfound.html if failure to upload occurse
 
 
 sub userfileupload {
-    my ($formname,$coursedoc,$subdir,$parser,$allfiles,$codebase,$destuname,$destudom)=@_;
+    my ($formname,$coursedoc,$subdir,$parser,$allfiles,$codebase,$destuname,
+        $destudom,$thumbwidth,$thumbheight)=@_;
     if (!defined($subdir)) { $subdir='unknown'; }
     my $fname=$env{'form.'.$formname.'.filename'};
     $fname=&clean_filename($fname);
@@ -1486,7 +1636,7 @@ sub userfileupload {
         if ($env{'form.folder'} =~ m/^(default|supplemental)/) {
             return &finishuserfileupload($docuname,$docudom,
 					 $formname,$fname,$parser,$allfiles,
-					 $codebase);
+					 $codebase,$thumbwidth,$thumbheight);
         } else {
             $fname=$env{'form.folder'}.'/'.$fname;
             return &process_coursefile('uploaddoc',$docuname,$docudom,
@@ -1496,8 +1646,9 @@ sub userfileupload {
     } elsif (defined($destuname)) {
         my $docuname=$destuname;
         my $docudom=$destudom;
-	return &finishuserfileupload($docuname,$docudom,$formname,
-				     $fname,$parser,$allfiles,$codebase);
+	return &finishuserfileupload($docuname,$docudom,$formname,$fname,
+				     $parser,$allfiles,$codebase,
+                                     $thumbwidth,$thumbheight);
         
     } else {
         my $docuname=$env{'user.name'};
@@ -1506,16 +1657,18 @@ sub userfileupload {
             $docuname=$env{'course.'.$env{'request.course.id'}.'.num'};
             $docudom=$env{'course.'.$env{'request.course.id'}.'.domain'};
         }
-	return &finishuserfileupload($docuname,$docudom,$formname,
-				     $fname,$parser,$allfiles,$codebase);
+	return &finishuserfileupload($docuname,$docudom,$formname,$fname,
+				     $parser,$allfiles,$codebase,
+                                     $thumbwidth,$thumbheight);
     }
 }
 
 sub finishuserfileupload {
-    my ($docuname,$docudom,$formname,$fname,$parser,$allfiles,$codebase) = @_;
+    my ($docuname,$docudom,$formname,$fname,$parser,$allfiles,$codebase,
+        $thumbwidth,$thumbheight) = @_;
     my $path=$docudom.'/'.$docuname.'/';
     my $filepath=$perlvar{'lonDocRoot'};
-    my ($fnamepath,$file);
+    my ($fnamepath,$file,$fetchthumb);
     $file=$fname;
     if ($fname=~m|/|) {
         ($fnamepath,$file) = ($fname =~ m|^(.*)/([^/]+)$|);
@@ -1551,11 +1704,28 @@ sub finishuserfileupload {
 		     ' for embedded media: '.$parse_result); 
         }
     }
+    if (($thumbwidth =~ /^\d+$/) && ($thumbheight =~ /^\d+$/)) {
+        my $input = $filepath.'/'.$file;
+        my $output = $filepath.'/'.'tn-'.$file;
+        my $thumbsize = $thumbwidth.'x'.$thumbheight;
+        system("convert -sample $thumbsize $input $output");
+        if (-e $filepath.'/'.'tn-'.$file) {
+            $fetchthumb  = 1; 
+        }
+    }
+ 
 # Notify homeserver to grep it
 #
     my $docuhome=&homeserver($docuname,$docudom);
     my $fetchresult= &reply('fetchuserfile:'.$path.$file,$docuhome);
     if ($fetchresult eq 'ok') {
+        if ($fetchthumb) {
+            my $thumbresult= &reply('fetchuserfile:'.$path.'tn-'.$file,$docuhome);
+            if ($thumbresult ne 'ok') {
+                &logthis('Failed to transfer '.$path.'tn-'.$file.' to host '.
+                         $docuhome.': '.$thumbresult);
+            }
+        }
 #
 # Return the URL to it
         return '/uploaded/'.$path.$file;
@@ -1563,7 +1733,7 @@ sub finishuserfileupload {
         &logthis('Failed to transfer '.$path.$file.' to host '.$docuhome.
 		 ': '.$fetchresult);
         return '/adm/notfound.html';
-    }    
+    }
 }
 
 sub extract_embedded_items {
@@ -1587,13 +1757,16 @@ sub extract_embedded_items {
     while (my $t=$p->get_token()) {
 	if ($t->[0] eq 'S') {
 	    my ($tagname, $attr) = ($t->[1],$t->[2]);
-	    push (@state, $tagname);
+	    push(@state, $tagname);
             if (lc($tagname) eq 'allow') {
                 &add_filetype($allfiles,$attr->{'src'},'src');
             }
 	    if (lc($tagname) eq 'img') {
 		&add_filetype($allfiles,$attr->{'src'},'src');
 	    }
+	    if (lc($tagname) eq 'a') {
+		&add_filetype($allfiles,$attr->{'href'},'href');
+	    }
             if (lc($tagname) eq 'script') {
                 if ($attr->{'archive'} =~ /\.jar$/i) {
                     &add_filetype($allfiles,$attr->{'archive'},'archive');
@@ -1688,6 +1861,12 @@ sub removeuserfile {
         if (($fname !~ /\.meta$/) && (&is_portfolio_file($fname))) {
             my $metafile = $fname.'.meta';
             my $metaresult = &removeuserfile($docuname,$docudom,$metafile); 
+	    my $url = "/uploaded/$docudom/$docuname/$fname";
+            my ($file,$group) = (&parse_portfolio_url($url))[3,4];
+            my $sqlresult = 
+                &update_portfolio_table($docuname,$docudom,$file,
+                                        'portfolio_metadata',$group,
+                                        'delete');
         }
     }
     return $result;
@@ -1710,6 +1889,12 @@ sub renameuserfile {
             my $newmeta = $new.'.meta';
             my $metaresult = 
                 &renameuserfile($docuname,$docudom,$oldmeta,$newmeta);
+	    my $url = "/uploaded/$docudom/$docuname/$old";
+            my ($file,$group) = (&parse_portfolio_url($url))[3,4];
+            my $sqlresult = 
+                &update_portfolio_table($docuname,$docudom,$file,
+                                        'portfolio_metadata',$group,
+                                        'delete');
         }
     }
     return $result;
@@ -1765,8 +1950,9 @@ sub flushcourselogs {
 # Write course id database (reverse lookup) to homeserver of courses 
 # Is used in pickcourse
 #
-    foreach my $crsid (keys(%courseidbuffer)) {
-        &courseidput($hostdom{$crsid},$courseidbuffer{$crsid},$crsid);
+    foreach my $crs_home (keys(%courseidbuffer)) {
+        &courseidput(&host_domain($crs_home),$courseidbuffer{$crs_home},
+		     $crs_home);
     }
 #
 # File accesses
@@ -1775,7 +1961,8 @@ sub flushcourselogs {
     foreach my $entry (keys(%accesshash)) {
         if ($entry =~ /___count$/) {
             my ($dom,$name);
-            ($dom,$name,undef)=($entry=~m:___(\w+)/(\w+)/(.*)___count$:);
+            ($dom,$name,undef)=
+		($entry=~m{___($match_domain)/($match_name)/(.*)___count$});
             if (! defined($dom) || $dom eq '' || 
                 ! defined($name) || $name eq '') {
                 my $cid = $env{'request.course.id'};
@@ -1796,7 +1983,7 @@ sub flushcourselogs {
                 }
             }
         } else {
-            my ($dom,$name) = ($entry=~m:___(\w+)/(\w+)/(.*)___(\w+)$:);
+            my ($dom,$name) = ($entry=~m{___($match_domain)/($match_name)/(.*)___(\w+)$});
             my %temphash=($entry => $accesshash{$entry});
             if (&put('nohist_resevaldata',\%temphash,$dom,$name) eq 'ok') {
                 delete $accesshash{$entry};
@@ -1832,13 +2019,12 @@ sub flushcourselogs {
         delete $domainrolehash{$entry};
     }
     foreach my $dom (keys(%domrolebuffer)) {
-        foreach my $tryserver (keys %libserv) {
-            if ($hostdom{$tryserver} eq $dom) {
-                unless (&reply('domroleput:'.$dom.':'.
-                  $domrolebuffer{$dom},$tryserver) eq 'ok') {
-                    &logthis('Put of domain roles failed for '.$dom.' and  '.$tryserver);
-                }
-            }
+	my %servers = &get_servers($dom,'library');
+	foreach my $tryserver (keys(%servers)) {
+	    unless (&reply('domroleput:'.$dom.':'.
+			   $domrolebuffer{$dom},$tryserver) eq 'ok') {
+		&logthis('Put of domain roles failed for '.$dom.' and  '.$tryserver);
+	    }
         }
     }
     $dumpcount++;
@@ -1972,21 +2158,61 @@ sub get_course_adv_roles {
 }
 
 sub get_my_roles {
-    my ($uname,$udom)=@_;
+    my ($uname,$udom,$context,$types,$roles,$roledoms)=@_;
     unless (defined($uname)) { $uname=$env{'user.name'}; }
     unless (defined($udom)) { $udom=$env{'user.domain'}; }
-    my %dumphash=
+    my %dumphash;
+    if ($context eq 'userroles') { 
+        %dumphash = &dump('roles',$udom,$uname);
+    } else {
+        %dumphash=
             &dump('nohist_userroles',$udom,$uname);
+    }
     my %returnhash=();
     my $now=time;
     foreach my $entry (keys(%dumphash)) {
-	my ($tend,$tstart)=split(/\:/,$dumphash{$entry});
+        my ($role,$tend,$tstart);
+        if ($context eq 'userroles') {
+	    ($role,$tend,$tstart)=split(/_/,$dumphash{$entry});
+        } else {
+            ($tend,$tstart)=split(/\:/,$dumphash{$entry});
+        }
         if (($tstart) && ($tstart<0)) { next; }
-        if (($tend) && ($tend<$now)) { next; }
-        if (($tstart) && ($now<$tstart)) { next; }
-        my ($role,$username,$domain,$section)=split(/\:/,$entry);
+        my $status = 'active';
+        if (($tend) && ($tend<$now)) {
+            $status = 'previous';
+        } 
+        if (($tstart) && ($now<$tstart)) {
+            $status = 'future';
+        }
+        if (ref($types) eq 'ARRAY') {
+            if (!grep(/^\Q$status\E$/,@{$types})) {
+                next;
+            } 
+        } else {
+            if ($status ne 'active') {
+                next;
+            }
+        }
+        my ($rolecode,$username,$domain,$section,$area);
+        if ($context eq 'userroles') {
+            ($area,$rolecode) = split(/_/,$entry);
+            (undef,$domain,$username,$section) = split(/\//,$area);
+        } else {
+            ($role,$username,$domain,$section) = split(/\:/,$entry);
+        }
+        if (ref($roledoms) eq 'ARRAY') {
+            if (!grep(/^\Q$domain\E$/,@{$roledoms})) {
+                next;
+            }
+        }
+        if (ref($roles) eq 'ARRAY') {
+            if (!grep(/^\Q$role\E$/,@{$roles})) {
+                next;
+            }
+        }
 	$returnhash{$username.':'.$domain.':'.$role}=$tstart.':'.$tend;
-     }
+    }
     return %returnhash;
 }
 
@@ -1996,7 +2222,7 @@ sub get_my_roles {
 
 sub postannounce {
     my ($server,$text)=@_;
-    unless (&allowed('psa',$hostdom{$server})) { return 'refused'; }
+    unless (&allowed('psa',&host_domain($server))) { return 'refused'; }
     unless ($text=~/\w/) { $text=''; }
     return &reply('setannounce:'.&escape($text),$server);
 }
@@ -2032,11 +2258,16 @@ sub courseiddump {
     my ($domfilter,$descfilter,$sincefilter,$instcodefilter,$ownerfilter,$coursefilter,$hostidflag,$hostidref,$typefilter,$regexp_ok)=@_;
     my %returnhash=();
     unless ($domfilter) { $domfilter=''; }
-    foreach my $tryserver (keys %libserv) {
-        if ( ($hostidflag == 1 && grep/^$tryserver$/,@{$hostidref}) || (!defined($hostidflag)) ) {
-	    if ((!$domfilter) || ($hostdom{$tryserver} eq $domfilter)) {
+    my %libserv = &all_library();
+    foreach my $tryserver (keys(%libserv)) {
+        if ( (  $hostidflag == 1 
+	        && grep(/^\Q$tryserver\E$/,@{$hostidref}) ) 
+	     || (!defined($hostidflag)) ) {
+
+	    if ($domfilter eq ''
+		|| (&host_domain($tryserver) eq $domfilter)) {
 	        foreach my $line (
-                 split(/\&/,&reply('courseiddump:'.$hostdom{$tryserver}.':'.
+                 split(/\&/,&reply('courseiddump:'.&host_domain($tryserver).':'.
 			       $sincefilter.':'.&escape($descfilter).':'.
                                &escape($instcodefilter).':'.&escape($ownerfilter).':'.&escape($coursefilter).':'.&escape($typefilter).':'.&escape($regexp_ok),
                                $tryserver))) {
@@ -2064,12 +2295,13 @@ sub dcmailput {
 sub dcmaildump {
     my ($dom,$startdate,$enddate,$senders) = @_;
     my %returnhash=();
-    if (exists($domain_primary{$dom})) {
+
+    if (defined(&domain($dom,'primary'))) {
         my $cmd='dcmaildump:'.$dom.':'.&escape($startdate).':'.
                                                          &escape($enddate).':';
 	my @esc_senders=map { &escape($_)} @$senders;
 	$cmd.=&escape(join('&',@esc_senders));
-	foreach my $line (split(/\&/,&reply($cmd,$domain_primary{$dom}))) {
+	foreach my $line (split(/\&/,&reply($cmd,&domain($dom,'primary')))) {
             my ($key,$value) = split(/\=/,$line,2);
             if (($key) && ($value)) {
                 $returnhash{&unescape($key)} = &unescape($value);
@@ -2090,19 +2322,19 @@ sub get_domain_roles {
     }
     my $rolelist = join(':',@{$roles});
     my %personnel = ();
-    foreach my $tryserver (keys(%libserv)) {
-        if ($hostdom{$tryserver} eq $dom) {
-            %{$personnel{$tryserver}}=();
-            foreach my $line (
-                split(/\&/,&reply('domrolesdump:'.$dom.':'.
-                   &escape($startdate).':'.&escape($enddate).':'.
-                   &escape($rolelist), $tryserver))) {
-                my ($key,$value) = split(/\=/,$line,2);
-                if (($key) && ($value)) {
-                    $personnel{$tryserver}{&unescape($key)} = &unescape($value);
-                }
-            }
-        }
+
+    my %servers = &get_servers($dom,'library');
+    foreach my $tryserver (keys(%servers)) {
+	%{$personnel{$tryserver}}=();
+	foreach my $line (split(/\&/,&reply('domrolesdump:'.$dom.':'.
+					    &escape($startdate).':'.
+					    &escape($enddate).':'.
+					    &escape($rolelist), $tryserver))) {
+	    my ($key,$value) = split(/\=/,$line,2);
+	    if (($key) && ($value)) {
+		$personnel{$tryserver}{&unescape($key)} = &unescape($value);
+	    }
+	}
     }
     return %personnel;
 }
@@ -2192,7 +2424,7 @@ sub checkin {
     my $now=time;
     my ($ta,$tb,$lonhost)=split(/\*/,$token);
     $lonhost=~tr/A-Z/a-z/;
-    my $dtoken=$ta.'_'.$hostname{$lonhost}.'_'.$tb;
+    my $dtoken=$ta.'_'.&hostname($lonhost).'_'.$tb;
     $dtoken=~s/\W/\_/g;
     my ($dummy,$tuname,$tudom,$tcrsid,$symb,$chtim,$rmaddr)=
                  split(/\&/,&unescape(&reply('tmpget:'.$dtoken,$lonhost)));
@@ -2735,6 +2967,7 @@ sub coursedescription {
     if (!$args->{'one_time'}) {
 	$envhash{'course.'.$normalid.'.last_cache'}=time;
     }
+
     if ($chome ne 'no_host') {
        %returnhash=&dump('environment',$cdomain,$cnum);
        if (!exists($returnhash{'con_lost'})) {
@@ -2810,8 +3043,8 @@ sub rolesinit {
 	    $area=~s/\_\w\w$//;
             my ($trole,$tend,$tstart,$group_privs);
 	    if ($role=~/^cr/) { 
-		if ($role=~m|^(cr/\w+/\w+/[a-zA-Z0-9]+)_(.*)$|) {
-		    ($trole,my $trest)=($role=~m|^(cr/\w+/\w+/[a-zA-Z0-9]+)_(.*)$|);
+		if ($role=~m|^(cr/$match_domain/$match_username/[a-zA-Z0-9]+)_(.*)$|) {
+		    ($trole,my $trest)=($role=~m|^(cr/$match_domain/$match_username/[a-zA-Z0-9]+)_(.*)$|);
 		    ($tend,$tstart)=split('_',$trest);
 		} else {
 		    $trole=$role;
@@ -2860,7 +3093,7 @@ sub custom_roleprivs {
     my ($allroles,$trole,$tdomain,$trest,$spec,$area) = @_;
     my ($rdummy,$rdomain,$rauthor,$rrole)=split(/\//,$trole);
     my $homsvr=homeserver($rauthor,$rdomain);
-    if ($hostname{$homsvr} ne '') {
+    if (&hostname($homsvr) ne '') {
         my ($rdummy,$roledef)=
             &get('roles',["rolesdef_$rrole"],$rdomain,$rauthor);
         if (($rdummy ne 'con_lost') && ($roledef ne '')) {
@@ -2890,7 +3123,7 @@ sub group_roleprivs {
     if (($tend!=0) && ($tend<$now)) { $access = 0; }
     if (($tstart!=0) && ($tstart>$now)) { $access=0; }
     if ($access) {
-        my ($course,$group) = ($area =~ m|(/\w+/\w+)/([^/]+)$|);
+        my ($course,$group) = ($area =~ m|(/$match_domain/$match_courseid)/([^/]+)$|);
         $$allgroups{$course}{$group} .=':'.$group_privs;
     }
 }
@@ -2921,7 +3154,7 @@ sub set_userprivs {
     if (keys(%{$allgroups}) > 0) {
         foreach my $role (keys %{$allroles}) {
             my ($trole,$area,$sec,$extendedarea);
-            if ($role =~ m-^(\w+|cr/\w+/\w+/\w+)\.(/\w+/\w+)(/?\w*)-) {
+            if ($role =~ m-^(\w+|cr/$match_domain/$match_username/\w+)\.(/$match_domain/$match_courseid)(/?\w*)\.-) {
                 $trole = $1;
                 $area = $2;
                 $sec = $3;
@@ -3033,7 +3266,23 @@ sub dump {
 
 sub dumpstore {
    my ($namespace,$udomain,$uname,$regexp,$range)=@_;
-   return &dump($namespace,$udomain,$uname,$regexp,$range);
+   if (!$udomain) { $udomain=$env{'user.domain'}; }
+   if (!$uname) { $uname=$env{'user.name'}; }
+   my $uhome=&homeserver($uname,$udomain);
+   if ($regexp) {
+       $regexp=&escape($regexp);
+   } else {
+       $regexp='.';
+   }
+   my $rep=&reply("dump:$udomain:$uname:$namespace:$regexp:$range",$uhome);
+   my @pairs=split(/\&/,$rep);
+   my %returnhash=();
+   foreach my $item (@pairs) {
+       my ($key,$value)=split(/=/,$item,2);
+       next if ($key =~ /^error: 2 /);
+       $returnhash{$key}=&thaw_unescape($value);
+   }
+   return %returnhash;
 }
 
 # -------------------------------------------------------------- keys interface
@@ -3046,6 +3295,7 @@ sub getkeys {
    my $rep=reply("keys:$udomain:$uname:$namespace",$uhome);
    my @keyarray=();
    foreach my $key (split(/\&/,$rep)) {
+      next if ($key =~ /^error: 2 /);
       push(@keyarray,&unescape($key));
    }
    return @keyarray;
@@ -3066,7 +3316,7 @@ sub currentdump {
    if ($rep eq "unknown_cmd") { 
        # an old lond will not know currentdump
        # Do a dump and make it look like a currentdump
-       my @tmp = &dump($courseid,$sdom,$sname,'.');
+       my @tmp = &dumpstore($courseid,$sdom,$sname,'.');
        return if ($tmp[0] =~ /^(error:|no_such_host)/);
        my %hash = @tmp;
        @tmp=();
@@ -3091,6 +3341,8 @@ sub convert_dump_to_currentdump{
     # we might run in to problems with parameter names =~ /^v\./
     while (my ($key,$value) = each(%hash)) {
         my ($v,$symb,$param) = split(/:/,$key);
+	$symb  = &unescape($symb);
+	$param = &unescape($param);
         next if ($v eq 'version' || $symb eq 'keys');
         next if (exists($returnhash{$symb}) &&
                  exists($returnhash{$symb}->{$param}) &&
@@ -3304,6 +3556,22 @@ sub portfolio_access {
     my ($requrl) = @_;
     my (undef,$udom,$unum,$file_name,$group) = &parse_portfolio_url($requrl);
     my $result = &get_portfolio_access($udom,$unum,$file_name,$group);
+    if ($result) {
+        my %setters;
+        if ($env{'user.name'} eq 'public' && $env{'user.domain'} eq 'public') {
+            my ($startblock,$endblock) =
+                &Apache::loncommon::blockcheck(\%setters,'port',$unum,$udom);
+            if ($startblock && $endblock) {
+                return 'B';
+            }
+        } else {
+            my ($startblock,$endblock) =
+                &Apache::loncommon::blockcheck(\%setters,'port');
+            if ($startblock && $endblock) {
+                return 'B';
+            }
+        }
+    }
     if ($result eq 'ok') {
        return 'F';
     } elsif ($result =~ /^[^:]+:guest_/) {
@@ -3367,9 +3635,16 @@ sub get_portfolio_access {
             }
             if (@users > 0) {
                 foreach my $userkey (@users) {
-                    if (exists($access_hash->{$userkey}{'users'}{$env{'user.name'}.':'.$env{'user.domain'}})) {
-                        return 'ok';
-                    }
+                    if (ref($access_hash->{$userkey}{'users'}) eq 'ARRAY') {
+                        foreach my $item (@{$access_hash->{$userkey}{'users'}}) {
+                            if (ref($item) eq 'HASH') {
+                                if (($item->{'uname'} eq $env{'user.name'}) &&
+                                    ($item->{'udom'} eq $env{'user.domain'})) {
+                                    return 'ok';
+                                }
+                            }
+                        }
+                    } 
                 }
             }
             my %roleshash;
@@ -3379,7 +3654,7 @@ sub get_portfolio_access {
                 my (%allgroups,%allroles); 
                 my ($start,$end,$role,$sec,$group);
                 foreach my $envkey (%env) {
-                    if ($envkey =~ m-^user\.role\.(gr|cc|in|ta|ep|st)\./([^/]+)/([^/]+)/?([^/]*)$-) {
+                    if ($envkey =~ m-^user\.role\.(gr|cc|in|ta|ep|st)\./($match_domain)/($match_courseid)/?([^/]*)$-) {
                         my $cid = $2.'_'.$3; 
                         if ($1 eq 'gr') {
                             $group = $4;
@@ -3392,7 +3667,7 @@ sub get_portfolio_access {
                             }
                             $allroles{$cid}{$1}{$sec} = $env{$envkey};
                         }
-                    } elsif ($envkey =~ m-^user\.role\./cr/(\w+/\w+/\w*)./([^/]+)/([^/]+)/?([^/]*)$-) {
+                    } elsif ($envkey =~ m-^user\.role\./cr/($match_domain/$match_username/\w*)./($match_domain)/($match_courseid)/?([^/]*)$-) {
                         my $cid = $2.'_'.$3;
                         if ($4 eq '') {
                             $sec = 'none';
@@ -3487,12 +3762,12 @@ sub parse_portfolio_url {
 
     my ($type,$udom,$unum,$group,$file_name);
     
-    if ($url =~  m-^/*uploaded/([^/]+)/([^/]+)/portfolio(/.+)$-) {
+    if ($url =~  m-^/*(?:uploaded|editupload)/($match_domain)/($match_username)/portfolio(/.+)$-) {
 	$type = 1;
         $udom = $1;
         $unum = $2;
         $file_name = $3;
-    } elsif ($url =~ m-^/*uploaded/([^/]+)/([^/]+)/groups/([^/]+)/portfolio/(.+)$-) {
+    } elsif ($url =~ m-^/*(?:uploaded|editupload)/($match_domain)/($match_courseid)/groups/([^/]+)/portfolio/(.+)$-) {
 	$type = 2;
         $udom = $1;
         $unum = $2;
@@ -3523,31 +3798,46 @@ sub is_portfolio_file {
 
 sub customaccess {
     my ($priv,$uri)=@_;
-    my ($urole,$urealm)=split(/\./,$env{'request.role'});
-    $urealm=~s/^\W//;
-    my ($udom,$ucrs,$usec)=split(/\//,$urealm);
+    my ($urole,$urealm)=split(/\./,$env{'request.role'},2);
+    my (undef,$udom,$ucrs,$usec)=split(/\//,$urealm);
+    $udom = &LONCAPA::clean_domain($udom);
+    $ucrs = &LONCAPA::clean_username($ucrs);
     my $access=0;
     foreach my $right (split(/\s*\,\s*/,&metadata($uri,'rule_rights'))) {
-	my ($effect,$realm,$role)=split(/\:/,$right);
-        if ($role) {
-	   if ($role ne $urole) { next; }
-        }
-        foreach my $scope (split(/\s*\,\s*/,$realm)) {
-            my ($tdom,$tcrs,$tsec)=split(/\_/,$scope);
-            if ($tdom) {
-		if ($tdom ne $udom) { next; }
-            }
-            if ($tcrs) {
-		if ($tcrs ne $ucrs) { next; }
-            }
-            if ($tsec) {
-		if ($tsec ne $usec) { next; }
-            }
-            $access=($effect eq 'allow');
-            last;
-        }
-	if ($realm eq '' && $role eq '') {
-            $access=($effect eq 'allow');
+	my ($effect,$realm,$role,$type)=split(/\:/,$right);
+	if ($type eq 'user') {
+	    foreach my $scope (split(/\s*\,\s*/,$realm)) {
+		my ($tdom,$tcrs)=split(/\_/,$scope);
+		if ($tdom) {
+		    if ($tdom ne $env{'user.domain'}) { next; }
+		}
+		if ($tcrs) {
+		    if ($tcrs ne $env{'user.name'}) { next; }
+		}
+		$access=($effect eq 'allow');
+		last;
+	    }
+	} else {
+	    if ($role) {
+		if ($role ne $urole) { next; }
+	    }
+	    foreach my $scope (split(/\s*\,\s*/,$realm)) {
+		my ($tdom,$tcrs,$tsec)=split(/\_/,$scope);
+		if ($tdom) {
+		    if ($tdom ne $udom) { next; }
+		}
+		if ($tcrs) {
+		    if ($tcrs ne $ucrs) { next; }
+		}
+		if ($tsec) {
+		    if ($tsec ne $usec) { next; }
+		}
+		$access=($effect eq 'allow');
+		last;
+	    }
+	    if ($realm eq '' && $role eq '') {
+		$access=($effect eq 'allow');
+	    }
 	}
     }
     return $access;
@@ -3556,12 +3846,21 @@ sub customaccess {
 # ------------------------------------------------- Check for a user privilege
 
 sub allowed {
-    my ($priv,$uri,$symb)=@_;
+    my ($priv,$uri,$symb,$role)=@_;
     my $ver_orguri=$uri;
     $uri=&deversion($uri);
     my $orguri=$uri;
     $uri=&declutter($uri);
-    
+
+    if ($priv eq 'evb') {
+# Evade communication block restrictions for specified role in a course
+        if ($env{'user.priv.'.$role} =~/evb\&([^\:]*)/) {
+            return $1;
+        } else {
+            return;
+        }
+    }
+
     if (defined($env{'allowed.'.$priv})) { return $env{'allowed.'.$priv}; }
 # Free bre access to adm and meta resources
     if (((($uri=~/^adm\//) && ($uri !~ m{/(?:smppg|bulletinboard)$})) 
@@ -3574,7 +3873,14 @@ sub allowed {
     my ($space,$domain,$name,@dir)=split('/',$uri);
     if (($space=~/^(uploaded|editupload)$/) && ($env{'user.name'} eq $name) && 
 	($env{'user.domain'} eq $domain) && ('portfolio' eq $dir[0])) {
-        return 'F';
+        my %setters;
+        my ($startblock,$endblock) = 
+            &Apache::loncommon::blockcheck(\%setters,'port');
+        if ($startblock && $endblock) {
+            return 'B';
+        } else {
+            return 'F';
+        }
     }
 
 # bre access to group portfolio for rgf priv in group, or mdg or vcg in course.
@@ -3850,6 +4156,8 @@ sub allowed {
     unless ($env{'request.course.id'}) {
 	if ($thisallowed eq 'A') {
 	    return 'A';
+        } elsif ($thisallowed eq 'B') {
+            return 'B';
 	} else {
 	    return '1';
 	}
@@ -3917,6 +4225,8 @@ sub allowed {
 
     if ($thisallowed eq 'A') {
 	return 'A';
+    } elsif ($thisallowed eq 'B') {
+        return 'B';
     }
    return 'F';
 }
@@ -4012,6 +4322,7 @@ sub definerole {
 sub metadata_query {
     my ($query,$custom,$customshow,$server_array)=@_;
     my %rhash;
+    my %libserv = &all_library();
     my @server_list = (defined($server_array) ? @$server_array
                                               : keys(%libserv) );
     for my $server (@server_list) {
@@ -4035,7 +4346,7 @@ sub log_query {
     my ($uname,$udom,$query,%filters)=@_;
     my $uhome=&homeserver($uname,$udom);
     if ($uhome eq 'no_host') { return 'error: no_host'; }
-    my $uhost=$hostname{$uhome};
+    my $uhost=&hostname($uhome);
     my $command=&escape(join(':',map{$_.'='.$filters{$_}} keys(%filters)));
     my $queryid=&reply("querysend:".$query.':'.$udom.':'.$uname.':'.$command,
                        $uhome);
@@ -4043,6 +4354,18 @@ sub log_query {
     return get_query_reply($queryid);
 }
 
+# -------------------------- Update MySQL table for portfolio file
+
+sub update_portfolio_table {
+    my ($uname,$udom,$file_name,$query,$group,$action) = @_;
+    my $homeserver = &homeserver($uname,$udom);
+    my $queryid=
+        &reply("querysend:".$query.':'.&escape($uname.':'.$udom.':'.$group).
+               ':'.&escape($file_name).':'.$action,$homeserver);
+    my $reply = &get_query_reply($queryid);
+    return $reply;
+}
+
 # ------- Request retrieval of institutional classlists for course(s)
 
 sub fetch_enrollment_query {
@@ -4055,7 +4378,7 @@ sub fetch_enrollment_query {
     } else {
         $homeserver = &homeserver($cnum,$dom);
     }
-    my $host=$hostname{$homeserver};
+    my $host=&hostname($homeserver);
     my $cmd = '';
     foreach my $affiliate (keys %{$affiliatesref}) {
         $cmd .= $affiliate.'='.join(",",@{$$affiliatesref{$affiliate}}).'%%';
@@ -4152,6 +4475,12 @@ sub courselog_query {
 }
 
 sub userlog_query {
+#
+# possible filters:
+# action: log check role
+# start: timestamp
+# end: timestamp
+#
     my ($uname,$udom,%filters)=@_;
     return &log_query($uname,$udom,'userlog',%filters);
 }
@@ -4160,8 +4489,18 @@ sub userlog_query {
 
 sub auto_run {
     my ($cnum,$cdom) = @_;
-    my $homeserver = &homeserver($cnum,$cdom);
-    my $response = &reply('autorun:'.$cdom,$homeserver);
+    my $response = 0;
+    my $settings;
+    my %domconfig = &get_dom('configuration',['autoenroll'],$cdom);
+    if (ref($domconfig{'autoenroll'}) eq 'HASH') {
+        $settings = $domconfig{'autoenroll'};
+        if ($settings->{'run'} eq '1') {
+            $response = 1;
+        }
+    } else {
+        my $homeserver = &homeserver($cnum,$cdom);
+        $response = &reply('autorun:'.$cdom,$homeserver);
+    }
     return $response;
 }
 
@@ -4191,15 +4530,27 @@ sub auto_validate_courseID {
 }
 
 sub auto_create_password {
-    my ($cnum,$cdom,$authparam) = @_;
-    my $homeserver = &homeserver($cnum,$cdom); 
+    my ($cnum,$cdom,$authparam,$udom) = @_;
+    my ($homeserver,$response);
     my $create_passwd = 0;
     my $authchk = '';
-    my $response=&unescape(&reply('autocreatepassword:'.$authparam.':'.$cdom,$homeserver));
-    if ($response eq 'refused') {
-        $authchk = 'refused';
+    if ($udom =~ /^$match_domain$/) {
+        $homeserver = &domain($udom,'primary');
+    }
+    if ($homeserver eq '') {
+        if (($cdom =~ /^$match_domain$/) && ($cnum =~ /^$match_courseid$/)) {
+            $homeserver = &homeserver($cnum,$cdom);
+        }
+    }
+    if ($homeserver eq '') {
+        $authchk = 'nodomain';
     } else {
-        ($authparam,$create_passwd,$authchk) = split/:/,$response;
+        $response=&unescape(&reply('autocreatepassword:'.$authparam.':'.$cdom,$homeserver));
+        if ($response eq 'refused') {
+            $authchk = 'refused';
+        } else {
+            ($authparam,$create_passwd,$authchk) = split/:/,$response;
+        }
     }
     return ($authparam,$create_passwd,$authchk);
 }
@@ -4246,7 +4597,7 @@ sub auto_photochoice {
 sub auto_photoupdate {
     my ($affiliatesref,$dom,$cnum,$photo) = @_;
     my $homeserver = &homeserver($cnum,$dom);
-    my $host=$hostname{$homeserver};
+    my $host=&hostname($homeserver);
     my $cmd = '';
     my $maxtries = 1;
     foreach my $affiliate (keys(%{$affiliatesref})) {
@@ -4286,12 +4637,11 @@ sub auto_instcode_format {
     my $courses = '';
     my @homeservers;
     if ($caller eq 'global') {
-        foreach my $tryserver (keys(%libserv)) {
-            if ($hostdom{$tryserver} eq $codedom) {
-                if (!grep(/^\Q$tryserver\E$/,@homeservers)) {
-                    push(@homeservers,$tryserver);
-                }
-            }
+	my %servers = &get_servers($codedom,'library');
+	foreach my $tryserver (keys(%servers)) {
+	    if (!grep(/^\Q$tryserver\E$/,@homeservers)) {
+		push(@homeservers,$tryserver);
+	    }
         }
     } else {
         push(@homeservers,&homeserver($caller,$codedom));
@@ -4325,35 +4675,31 @@ sub auto_instcode_format {
 sub auto_instcode_defaults {
     my ($domain,$returnhash,$code_order) = @_;
     my @homeservers;
-    foreach my $tryserver (keys(%libserv)) {
-        if ($hostdom{$tryserver} eq $domain) {
-            if (!grep(/^\Q$tryserver\E$/,@homeservers)) {
-                push(@homeservers,$tryserver);
-            }
-        }
+
+    my %servers = &get_servers($domain,'library');
+    foreach my $tryserver (keys(%servers)) {
+	if (!grep(/^\Q$tryserver\E$/,@homeservers)) {
+	    push(@homeservers,$tryserver);
+	}
     }
-    my $ok_response = 0;
+
     my $response;
-    while (@homeservers > 0 && $ok_response == 0) {
-        my $server = shift(@homeservers);
+    foreach my $server (@homeservers) {
         $response=&reply('autoinstcodedefaults:'.$domain,$server);
-        if ($response !~ /(con_lost|error|no_such_host|refused)/) {
-            foreach my $pair (split(/\&/,$response)) {
-                my ($name,$value)=split(/\=/,$pair);
-                if ($name eq 'code_order') {
-                    @{$code_order} = split(/\&/,&unescape($value));
-                } else {
-                    $returnhash->{&unescape($name)}=&unescape($value);
-                }
-            }
-            $ok_response = 1;
-        }
-    }
-    if ($ok_response) {
-        return 'ok';
-    } else {
-        return $response;
+        next if ($response =~ /(con_lost|error|no_such_host|refused)/);
+	
+	foreach my $pair (split(/\&/,$response)) {
+	    my ($name,$value)=split(/\=/,$pair);
+	    if ($name eq 'code_order') {
+		@{$code_order} = split(/\&/,&unescape($value));
+	    } else {
+		$returnhash->{&unescape($name)}=&unescape($value);
+	    }
+	}
+	return 'ok';
     }
+
+    return $response;
 } 
 
 sub auto_validate_class_sec {
@@ -4367,13 +4713,8 @@ sub auto_validate_class_sec {
 # ------------------------------------------------------- Course Group routines
 
 sub get_coursegroups {
-    my ($cdom,$cnum,$group) = @_;
-    return(&dump('coursegroups',$cdom,$cnum,$group));
-}
-
-sub get_deleted_groups {
-    my ($cdom,$cnum,$group) = @_;
-    return(&dump('deleted_groups',$cdom,$cnum,$group));
+    my ($cdom,$cnum,$group,$namespace) = @_;
+    return(&dump($namespace,$cdom,$cnum,$group));
 }
 
 sub modify_coursegroup {
@@ -4381,18 +4722,26 @@ sub modify_coursegroup {
     return(&put('coursegroups',$groupsettings,$cdom,$cnum));
 }
 
-sub delete_coursegroup {
-    my ($cdom,$cnum,$group) = @_;
-    my %curr_group = &get_coursegroups($cdom,$cnum,$group);
+sub toggle_coursegroup_status {
+    my ($cdom,$cnum,$group,$action) = @_;
+    my ($from_namespace,$to_namespace);
+    if ($action eq 'delete') {
+        $from_namespace = 'coursegroups';
+        $to_namespace = 'deleted_groups';
+    } else {
+        $from_namespace = 'deleted_groups';
+        $to_namespace = 'coursegroups';
+    }
+    my %curr_group = &get_coursegroups($cdom,$cnum,$group,$from_namespace);
     if (my $tmp = &error(%curr_group)) {
         &Apache::lonnet::logthis('Error retrieving group: '.$tmp.' in '.$cnum.':'.$cdom);
         return ('read error',$tmp);
     } else {
         my %savedsettings = %curr_group; 
-        my $result = &put('deleted_groups',\%savedsettings,$cdom,$cnum);
+        my $result = &put($to_namespace,\%savedsettings,$cdom,$cnum);
         my $deloutcome;
         if ($result eq 'ok') {
-            $deloutcome = &del('coursegroups',[$group],$cdom,$cnum);
+            $deloutcome = &del($from_namespace,[$group],$cdom,$cnum);
         } else {
             return ('write error',$result);
         }
@@ -4427,7 +4776,7 @@ sub get_active_groups {
     my $now = time;
     my %groups = ();
     foreach my $key (keys(%env)) {
-        if ($key =~ m-user\.role\.gr\./([^/]+)/([^/]+)/(\w+)$-) {
+        if ($key =~ m-user\.role\.gr\./($match_domain)/($match_courseid)/(\w+)$-) {
             my ($start,$end) = split(/\./,$env{$key});
             if (($end!=0) && ($end<$now)) { next; }
             if (($start!=0) && ($start>$now)) { next; }
@@ -4448,8 +4797,6 @@ sub get_users_groups {
     my ($udom,$uname,$courseid) = @_;
     my @usersgroups;
     my $cachetime=1800;
-    $courseid=~s/\_/\//g;
-    $courseid=~s/^(\w)/\/$1/;
 
     my $hashid="$udom:$uname:$courseid";
     my ($grouplist,$cached)=&is_cached_new('getgroups',$hashid);
@@ -4457,38 +4804,34 @@ sub get_users_groups {
         @usersgroups = split(/:/,$grouplist);
     } else {  
         $grouplist = '';
-        my %roleshash = &dump('roles',$udom,$uname,$courseid);
-        my ($tmp) = keys(%roleshash);
-        if ($tmp=~/^error:/) {
-            &logthis('Error retrieving roles: '.$tmp.' for '.$uname.':'.$udom);
-        } else {
-            my $access_end = $env{'course.'.$courseid.
-                                  '.default_enrollment_end_date'};
-            my $now = time;
-            foreach my $key (keys(%roleshash)) {
-                if ($key =~ /^\Q$courseid\E\/(\w+)\_gr$/) {
-                    my $group = $1;
-                    if ($roleshash{$key} =~ /_(\d+)_(\d+)$/) {
-                        my $start = $2;
-                        my $end = $1;
-                        if ($start == -1) { next; } # deleted from group
-                        if (($start!=0) && ($start>$now)) { next; }
-                        if (($end!=0) && ($end<$now)) {
-                            if ($access_end && $access_end < $now) {
-                                if ($access_end - $end < 86400) {
-                                    push(@usersgroups,$group);
-                                }
+        my $courseurl = &courseid_to_courseurl($courseid);
+        my %roleshash = &dump('roles',$udom,$uname,$courseurl);
+        my $access_end = $env{'course.'.$courseid.
+                              '.default_enrollment_end_date'};
+        my $now = time;
+        foreach my $key (keys(%roleshash)) {
+            if ($key =~ /^\Q$courseurl\E\/(\w+)\_gr$/) {
+                my $group = $1;
+                if ($roleshash{$key} =~ /_(\d+)_(\d+)$/) {
+                    my $start = $2;
+                    my $end = $1;
+                    if ($start == -1) { next; } # deleted from group
+                    if (($start!=0) && ($start>$now)) { next; }
+                    if (($end!=0) && ($end<$now)) {
+                        if ($access_end && $access_end < $now) {
+                            if ($access_end - $end < 86400) {
+                                push(@usersgroups,$group);
                             }
-                            next;
                         }
-                        push(@usersgroups,$group);
+                        next;
                     }
+                    push(@usersgroups,$group);
                 }
             }
-            @usersgroups = &sort_course_groups($courseid,@usersgroups);
-            $grouplist = join(':',@usersgroups);
-            &do_cache_new('getgroups',$hashid,$grouplist,$cachetime);
         }
+        @usersgroups = &sort_course_groups($courseid,@usersgroups);
+        $grouplist = join(':',@usersgroups);
+        &do_cache_new('getgroups',$hashid,$grouplist,$cachetime);
     }
     return @usersgroups;
 }
@@ -4496,8 +4839,7 @@ sub get_users_groups {
 sub devalidate_getgroups_cache {
     my ($udom,$uname,$cdom,$cnum)=@_;
     my $courseid = $cdom.'_'.$cnum;
-    $courseid=~s/\_/\//g;
-    $courseid=~s/^(\w)/\/$1/;
+
     my $hashid="$udom:$uname:$courseid";
     &devalidate_cache_new('getgroups',$hashid);
 }
@@ -4536,7 +4878,7 @@ sub assignrole {
     my $mrole;
     if ($role =~ /^cr\//) {
         my $cwosec=$url;
-        $cwosec=~s/^\/(\w+)\/(\w+)\/.*/$1\/$2/;
+        $cwosec=~s/^\/($match_domain)\/($match_courseid)\/.*/$1\/$2/;
 	unless (&allowed('ccr',$cwosec)) {
            &logthis('Refused custom assignrole: '.
              $udom.' '.$uname.' '.$url.' '.$role.' '.$end.' '.$start.' by '.
@@ -4546,7 +4888,7 @@ sub assignrole {
         $mrole='cr';
     } elsif ($role =~ /^gr\//) {
         my $cwogrp=$url;
-        $cwogrp=~s/^\/(\w+)\/(\w+)\/.*/$1\/$2/;
+        $cwogrp=~s{^/($match_domain)/($match_courseid)/.*}{$1/$2};
         unless (&allowed('mdg',$cwogrp)) {
             &logthis('Refused group assignrole: '.
               $udom.' '.$uname.' '.$url.' '.$role.' '.$end.' '.$start.' by '.
@@ -4556,7 +4898,7 @@ sub assignrole {
         $mrole='gr';
     } else {
         my $cwosec=$url;
-        $cwosec=~s/^\/(\w+)\/(\w+)\/.*/$1\/$2/;
+        $cwosec=~s/^\/($match_domain)\/($match_courseid)\/.*/$1\/$2/;
         unless ((&allowed('c'.$role,$cwosec)) || &allowed('c'.$role,$udom)) { 
            &logthis('Refused assignrole: '.
              $udom.' '.$uname.' '.$url.' '.$role.' '.$end.' '.$start.' by '.
@@ -4636,8 +4978,8 @@ sub modifyuser {
         $umode,   $upass, $first,
         $middle,  $last,  $gene,
         $forceid, $desiredhome, $email)=@_;
-    $udom=~s/\W//g;
-    $uname=~s/\W//g;
+    $udom= &LONCAPA::clean_domain($udom);
+    $uname=&LONCAPA::clean_username($uname);
     &logthis('Call to modify user '.$udom.', '.$uname.', '.$uid.', '.
              $umode.', '.$first.', '.$middle.', '.
 	     $last.', '.$gene.'(forceid: '.$forceid.')'.
@@ -4650,21 +4992,19 @@ sub modifyuser {
     if (($uhome eq 'no_host') && 
 	(($umode && $upass) || ($umode eq 'localauth'))) {
         my $unhome='';
-        if (defined($desiredhome) && $hostdom{$desiredhome} eq $udom) { 
+        if (defined($desiredhome) && &host_domain($desiredhome) eq $udom) { 
             $unhome = $desiredhome;
 	} elsif($env{'course.'.$env{'request.course.id'}.'.domain'} eq $udom) {
 	    $unhome=$env{'course.'.$env{'request.course.id'}.'.home'};
         } else { # load balancing routine for determining $unhome
-            my $tryserver;
             my $loadm=10000000;
-            foreach $tryserver (keys %libserv) {
-	       if ($hostdom{$tryserver} eq $udom) {
-                  my $answer=reply('load',$tryserver);
-                  if (($answer=~/\d+/) && ($answer<$loadm)) {
-		      $loadm=$answer;
-                      $unhome=$tryserver;
-                  }
-	       }
+	    my %servers = &get_servers($udom,'library');
+	    foreach my $tryserver (keys(%servers)) {
+		my $answer=reply('load',$tryserver);
+		if (($answer=~/\d+/) && ($answer<$loadm)) {
+		    $loadm=$answer;
+		    $unhome=$tryserver;
+		}
 	    }
         }
         if (($unhome eq '') || ($unhome eq 'no_host')) {
@@ -4879,7 +5219,7 @@ sub createcourse {
    }
 # ------------------------------------------------ Check supplied server name
     $course_server = $env{'user.homeserver'} if (! defined($course_server));
-    if (! exists($libserv{$course_server})) {
+    if (! &is_library($course_server)) {
         return 'error:bad server name '.$course_server;
     }
 # ------------------------------------------------------------- Make the course
@@ -4922,6 +5262,16 @@ ENDINITMAP
     return '/'.$udom.'/'.$uname;
 }
 
+sub is_course {
+    my ($cdom,$cnum) = @_;
+    my %courses = &courseiddump($cdom,'.',1,'.','.',$cnum,undef,
+				undef,'.');
+    if (exists($courses{$cdom.'_'.$cnum})) {
+        return 1;
+    }
+    return 0;
+}
+
 # ---------------------------------------------------------- Assign Custom Role
 
 sub assigncustomrole {
@@ -4980,9 +5330,7 @@ sub is_locked {
 
 sub declutter_portfile {
     my ($file) = @_;
-    &logthis("got $file");
-    $file =~ s-^(/portfolio/|portfolio/)-/-;
-    &logthis("ret $file");
+    $file =~ s{^(/portfolio/|portfolio/)}{/};
     return $file;
 }
 
@@ -5007,7 +5355,7 @@ sub save_selected_files {
     my ($user, $path, @files) = @_;
     my $filename = $user."savedfiles";
     my @other_files = &files_not_in_path($user, $path);
-    open (OUT, '>'.$Apache::lonnet::perlvar{'lonDaemons'}.'/tmp/'.$filename);
+    open (OUT, '>'.$tmpdir.$filename);
     foreach my $file (@files) {
         print (OUT $env{'form.currentpath'}.$file."\n");
     }
@@ -5200,12 +5548,68 @@ sub modify_access_controls {
         #  remove lock
         my @del_lock = ($file_name."\0".'locked_access_records');
         my $dellockoutcome = &del('file_permissions',\@del_lock,$domain,$user);
+        my ($file,$group);
+        if (&is_course($domain,$user)) {
+            ($group,$file) = split(/\//,$file_name,2);
+        } else {
+            $file = $file_name;
+        }
+        my $sqlresult =
+            &update_portfolio_table($user,$domain,$file,'portfolio_access',
+                                    $group);
     } else {
         $outcome = "error: could not obtain lockfile\n";  
     }
     return ($outcome,$deloutcome,\%new_values,\%translation);
 }
 
+sub make_public_indefinitely {
+    my ($requrl) = @_;
+    my $now = time;
+    my $action = 'activate';
+    my $aclnum = 0;
+    if (&is_portfolio_url($requrl)) {
+        my (undef,$udom,$unum,$file_name,$group) =
+            &parse_portfolio_url($requrl);
+        my $current_perms = &get_portfile_permissions($udom,$unum);
+        my %access_controls = &get_access_controls($current_perms,
+                                                   $group,$file_name);
+        foreach my $key (keys(%{$access_controls{$file_name}})) {
+            my ($num,$scope,$end,$start) = 
+                ($key =~ /^([^:]+):([a-z]+)_(\d*)_?(\d*)$/);
+            if ($scope eq 'public') {
+                if ($start <= $now && $end == 0) {
+                    $action = 'none';
+                } else {
+                    $action = 'update';
+                    $aclnum = $num;
+                }
+                last;
+            }
+        }
+        if ($action eq 'none') {
+             return 'ok';
+        } else {
+            my %changes;
+            my $newend = 0;
+            my $newstart = $now;
+            my $newkey = $aclnum.':public_'.$newend.'_'.$newstart;
+            $changes{$action}{$newkey} = {
+                type => 'public',
+                time => {
+                    start => $newstart,
+                    end   => $newend,
+                },
+            };
+            my ($outcome,$deloutcome,$new_values,$translation) =
+                &modify_access_controls($file_name,\%changes,$udom,$unum);
+            return $outcome;
+        }
+    } else {
+        return 'invalid';
+    }
+}
+
 #------------------------------------------------------Get Marked as Read Only
 
 sub get_marked_as_readonly {
@@ -5348,28 +5752,27 @@ sub dirlist {
             return @listing_results;
         } elsif(!defined($alternateDirectoryRoot)) {
             my %allusers;
-            foreach my $tryserver (keys(%libserv)) {
-                if($hostdom{$tryserver} eq $udom) {
-                    my $listing = &reply('ls2:'.$perlvar{'lonDocRoot'}.'/res/'.
-					 $udom, $tryserver);
-                    my @listing_results;
-                    if ($listing eq 'unknown_cmd') {
-                        $listing = &reply('ls:'.$perlvar{'lonDocRoot'}.'/res/'.
-					  $udom, $tryserver);
-                        @listing_results = split(/:/,$listing);
-                    } else {
-                        @listing_results =
-                            map { &unescape($_); } split(/:/,$listing);
-                    }
-                    if ($listing_results[0] ne 'no_such_dir' && 
-                        $listing_results[0] ne 'empty'       &&
-                        $listing_results[0] ne 'con_lost') {
-                        foreach my $line (@listing_results) {
-                            my ($entry) = split(/&/,$line,2);
-                            $allusers{$entry} = 1;
-                        }
-                    }
-                }
+	    my %servers = &get_servers($udom,'library');
+	    foreach my $tryserver (keys(%servers)) {
+		my $listing = &reply('ls2:'.$perlvar{'lonDocRoot'}.'/res/'.
+				     $udom, $tryserver);
+		my @listing_results;
+		if ($listing eq 'unknown_cmd') {
+		    $listing = &reply('ls:'.$perlvar{'lonDocRoot'}.'/res/'.
+				      $udom, $tryserver);
+		    @listing_results = split(/:/,$listing);
+		} else {
+		    @listing_results =
+			map { &unescape($_); } split(/:/,$listing);
+		}
+		if ($listing_results[0] ne 'no_such_dir' && 
+		    $listing_results[0] ne 'empty'       &&
+		    $listing_results[0] ne 'con_lost') {
+		    foreach my $line (@listing_results) {
+			my ($entry) = split(/&/,$line,2);
+			$allusers{$entry} = 1;
+		    }
+		}
             }
             my $alluserstr='';
             foreach my $user (sort(keys(%allusers))) {
@@ -5381,18 +5784,12 @@ sub dirlist {
             return ('missing user name');
         }
     } elsif(!defined($alternateDirectoryRoot)) {
-        my $tryserver;
-        my %alldom=();
-        foreach $tryserver (keys(%libserv)) {
-            $alldom{$hostdom{$tryserver}}=1;
-        }
-        my $alldomstr='';
-        foreach my $domain (sort(keys(%alldom))) {
-            $alldomstr.=$perlvar{'lonDocRoot'}.'/res/'.$domain.'/&domain:';
-        }
-        $alldomstr=~s/:$//;
-        return split(/:/,$alldomstr);       
-    } else {
+        my @all_domains = sort(&all_domains());
+         foreach my $domain (@all_domains) {
+             $domain = $perlvar{'lonDocRoot'}.'/res/'.$domain.'/&domain';
+         }
+         return @all_domains;
+     } else {
         return ('missing domain');
     }
 }
@@ -5411,8 +5808,8 @@ sub dirlist {
 ##
 sub GetFileTimestamp {
     my ($studentDomain,$studentName,$filename,$root)=@_;
-    $studentDomain=~s/\W//g;
-    $studentName=~s/\W//g;
+    $studentDomain = &LONCAPA::clean_domain($studentDomain);
+    $studentName   = &LONCAPA::clean_username($studentName);
     my $subdir=$studentName.'__';
     $subdir =~ s/(.)(.)(.).*/$1\/$2\/$3/;
     my $proname="$studentDomain/$subdir/$studentName";
@@ -5435,13 +5832,13 @@ sub stat_file {
     my ($udom,$uname,$file,$dir);
     if ($uri =~ m-^/(uploaded|editupload)/-) {
 	($udom,$uname,$file) =
-	    ($uri =~ m-/(?:uploaded|editupload)/?([^/]*)/?([^/]*)/?(.*)-);
+	    ($uri =~ m-/(?:uploaded|editupload)/?($match_domain)/?($match_name)/?(.*)-);
 	$file = 'userfiles/'.$file;
 	$dir = &propath($udom,$uname);
     }
     if ($uri =~ m-^/res/-) {
 	($udom,$uname) = 
-	    ($uri =~ m-/(?:res)/?([^/]*)/?([^/]*)/-);
+	    ($uri =~ m-/(?:res)/?($match_domain)/?($match_username)/-);
 	$file = $uri;
     }
 
@@ -5550,6 +5947,13 @@ sub devalidatecourseresdata {
 
 
 # --------------------------------------------------- Course Resourcedata Query
+#
+#  Parameters:
+#      $coursenum    - Number of the course.
+#      $coursedomain - Domain at which the course was created.
+#  Returns:
+#     A hash of the course parameters along (I think) with timestamps
+#     and version info.
 
 sub get_courseresdata {
     my ($coursenum,$coursedomain)=@_;
@@ -5608,7 +6012,21 @@ sub get_userresdata {
     }
     return $tmp;
 }
-
+#----------------------------------------------- resdata - return resource data
+#  Purpose:
+#    Return resource data for either users or for a course.
+#  Parameters:
+#     $name      - Course/user name.
+#     $domain    - Name of the domain the user/course is registered on.
+#     $type      - Type of thing $name is (must be 'course' or 'user'
+#     @which     - Array of names of resources desired.
+#  Returns:
+#     The value of the first reasource in @which that is found in the
+#     resource hash.
+#  Exceptional Conditions:
+#     If the $type passed in is not valid (not the string 'course' or 
+#     'user', an undefined  reference is returned.
+#     If none of the resources are found, an undef is returned
 sub resdata {
     my ($name,$domain,$type,@which)=@_;
     my $result;
@@ -5955,7 +6373,8 @@ sub packages_tab_default {
 	    $do_default=1;
 	} elsif ($pack_type eq 'extension') {
 	    push(@extension,[$package,$pack_type,$pack_part]);
-	} else {
+	} elsif ($pack_part eq $part || $pack_type eq 'part') {
+	    # only look at packages defaults for packages that this id is
 	    push(@specifics,[$package,$pack_type,$pack_part]);
 	}
     }
@@ -6022,7 +6441,7 @@ sub metadata {
 	(($uri =~ m|^/*adm/|) && 
 	     ($uri !~ m|^adm/includes|) && ($uri !~ m|/bulletinboard$|)) ||
         ($uri =~ m|/$|) || ($uri =~ m|/.meta$|) || ($uri =~ /^~/) ||
-	($uri =~ m|home/[^/]+/public_html/|)) {
+	($uri =~ m|home/$match_username/public_html/|)) {
 	return undef;
     }
     my $filename=$uri;
@@ -6171,13 +6590,18 @@ sub metadata {
 	    }
 	}
 	my ($extension) = ($uri =~ /\.(\w+)$/);
+	$extension = lc($extension);
+	if ($extension eq 'htm') { $extension='html'; }
+
 	foreach my $key (keys(%packagetab)) {
 	    #no specific packages #how's our extension
 	    if ($key!~/^extension_\Q$extension\E&/) { next; }
 	    &metadata_create_package_def($uri,$key,'extension_'.$extension,
 					 \%metathesekeys);
 	}
-	if (!exists($metaentry{':packages'})) {
+
+	if (!exists($metaentry{':packages'})
+	    || $packagetab{"import_defaults&extension_$extension"}) {
 	    foreach my $key (keys(%packagetab)) {
 		#no specific packages well let's get default then
 		if ($key!~/^default&/) { next; }
@@ -6673,7 +7097,6 @@ sub getCODE {
 
 sub rndseed {
     my ($symb,$courseid,$domain,$username)=@_;
-
     my ($wsymb,$wcourseid,$wdomain,$wusername)=&whichuser();
     if (!$symb) {
 	unless ($symb=$wsymb) { return time; }
@@ -6877,13 +7300,14 @@ sub setup_random_from_rndseed {
 }
 
 sub latest_receipt_algorithm_id {
-    return 'receipt2';
+    return 'receipt3';
 }
 
 sub recunique {
     my $fucourseid=shift;
     my $unique;
-    if ($env{"course.$fucourseid.receiptalg"} eq 'receipt2') {
+    if ($env{"course.$fucourseid.receiptalg"} eq 'receipt2' ||
+	$env{"course.$fucourseid.receiptalg"} eq 'receipt3' ) {
 	$unique=$env{"course.$fucourseid.internal.encseed"};
     } else {
 	$unique=$perlvar{'lonReceipt'};
@@ -6894,7 +7318,8 @@ sub recunique {
 sub recprefix {
     my $fucourseid=shift;
     my $prefix;
-    if ($env{"course.$fucourseid.receiptalg"} eq 'receipt2') {
+    if ($env{"course.$fucourseid.receiptalg"} eq 'receipt2'||
+	$env{"course.$fucourseid.receiptalg"} eq 'receipt3' ) {
 	$prefix=$env{"course.$fucourseid.internal.encpref"};
     } else {
 	$prefix=$perlvar{'lonHostID'};
@@ -6904,15 +7329,23 @@ sub recprefix {
 
 sub ireceipt {
     my ($funame,$fudom,$fucourseid,$fusymb,$part)=@_;
+
+    my $return =&recprefix($fucourseid).'-';
+
+    if ($env{"course.$fucourseid.receiptalg"} eq 'receipt3' ||
+	$env{'request.state'} eq 'construct') {
+	$return .= (&digest("$funame,$fudom,$fucourseid,$fusymb,$part")%10000);
+	return $return;
+    }
+
     my $cuname=unpack("%32C*",$funame);
     my $cudom=unpack("%32C*",$fudom);
     my $cucourseid=unpack("%32C*",$fucourseid);
     my $cusymb=unpack("%32C*",$fusymb);
     my $cunique=&recunique($fucourseid);
     my $cpart=unpack("%32S*",$part);
-    my $return =&recprefix($fucourseid).'-';
-    if ($env{"course.$fucourseid.receiptalg"} eq 'receipt2' ||
-	$env{'request.state'} eq 'construct') {
+    if ($env{"course.$fucourseid.receiptalg"} eq 'receipt2') {
+
 	#&logthis("doing receipt2  using parts $cpart, uname $cuname and udom $cudom gets  ".($cpart%$cuname)." and ".($cpart%$cudom));
 			       
 	$return.= ($cunique%$cuname+
@@ -7000,61 +7433,60 @@ sub repcopy_userfile {
     if ($file =~ m -^/*(uploaded|editupload)/-) { $file=&filelocation("",$file); }
     if ($file =~ m|^/home/httpd/html/lonUsers/|) { return 'ok'; }
     my ($cdom,$cnum,$filename) = 
-	($file=~m|^\Q$perlvar{'lonDocRoot'}\E/+userfiles/+([^/]+)/+([^/]+)/+(.*)|);
-    my ($info,$rtncode);
+	($file=~m|^\Q$perlvar{'lonDocRoot'}\E/+userfiles/+($match_domain)/+($match_name)/+(.*)|);
     my $uri="/uploaded/$cdom/$cnum/$filename";
     if (-e "$file") {
+# we already have a local copy, check it out
 	my @fileinfo = stat($file);
+	my $rtncode;
+	my $info;
 	my $lwpresp = &getuploaded('HEAD',$uri,$cdom,$cnum,\$info,\$rtncode);
 	if ($lwpresp ne 'ok') {
+# there is no such file anymore, even though we had a local copy
 	    if ($rtncode eq '404') {
 		unlink($file);
 	    }
-	    #my $ua=new LWP::UserAgent;
-	    #my $request=new HTTP::Request('GET',&tokenwrapper($uri));
-	    #my $response=$ua->request($request);
-	    #if ($response->is_success()) {
-	#	return $response->content;
-	#    } else {
-	#	return -1;
-	#    }
 	    return -1;
 	}
 	if ($info < $fileinfo[9]) {
+# nice, the file we have is up-to-date, just say okay
 	    return 'ok';
+	} else {
+# the file is outdated, get rid of it
+	    unlink($file);
 	}
-	$info = '';
-	$lwpresp = &getuploaded('GET',$uri,$cdom,$cnum,\$info,\$rtncode);
-	if ($lwpresp ne 'ok') {
-	    return -1;
-	}
-    } else {
-	my $lwpresp = &getuploaded('GET',$uri,$cdom,$cnum,\$info,\$rtncode);
-	if ($lwpresp ne 'ok') {
-	    my $ua=new LWP::UserAgent;
-	    my $request=new HTTP::Request('GET',&tokenwrapper($uri));
-	    my $response=$ua->request($request);
-	    if ($response->is_success()) {
-		$info=$response->content;
-	    } else {
-		return -1;
-	    }
-	}
-	my @parts = ($cdom,$cnum); 
-	if ($filename =~ m|^(.+)/[^/]+$|) {
-	    push @parts, split(/\//,$1);
-	}
-	my $path = $perlvar{'lonDocRoot'}.'/userfiles';
-	foreach my $part (@parts) {
-	    $path .= '/'.$part;
-	    if (!-e $path) {
-		mkdir($path,0770);
-	    }
+    }
+# one way or the other, at this point, we don't have the file
+# construct the correct path for the file
+    my @parts = ($cdom,$cnum); 
+    if ($filename =~ m|^(.+)/[^/]+$|) {
+	push @parts, split(/\//,$1);
+    }
+    my $path = $perlvar{'lonDocRoot'}.'/userfiles';
+    foreach my $part (@parts) {
+	$path .= '/'.$part;
+	if (!-e $path) {
+	    mkdir($path,0770);
 	}
     }
-    open(FILE,">$file");
-    print FILE $info;
-    close(FILE);
+# now the path exists for sure
+# get a user agent
+    my $ua=new LWP::UserAgent;
+    my $transferfile=$file.'.in.transfer';
+# FIXME: this should flock
+    if (-e $transferfile) { return 'ok'; }
+    my $request;
+    $uri=~s/^\///;
+    $request=new HTTP::Request('GET','http://'.&hostname(&homeserver($cnum,$cdom)).'/raw/'.$uri);
+    my $response=$ua->request($request,$transferfile);
+# did it work?
+    if ($response->is_error()) {
+	unlink($transferfile);
+	&logthis("Userfile repcopy failed for $uri");
+	return -1;
+    }
+# worked, rename the transfer file
+    rename($transferfile,$file);
     return 'ok';
 }
 
@@ -7068,7 +7500,7 @@ sub tokenwrapper {
     if ($udom && $uname && $file) {
 	$file=~s|(\?\.*)*$||;
         &appenv("userfile.$udom/$uname/$file" => $env{'request.course.id'});
-        return 'http://'.$hostname{ &homeserver($uname,$udom)}.'/'.$uri.
+        return 'http://'.&hostname(&homeserver($uname,$udom)).'/'.$uri.
                (($uri=~/\?/)?'&':'?').'token='.$token.
                                '&tokenissued='.$perlvar{'lonHostID'};
     } else {
@@ -7076,10 +7508,14 @@ sub tokenwrapper {
     }
 }
 
+# call with reqtype HEAD: get last modification time
+# call with reqtype GET: get the file contents
+# Do not call this with reqtype GET for large files! It loads everything into memory
+#
 sub getuploaded {
     my ($reqtype,$uri,$cdom,$cnum,$info,$rtncode) = @_;
     $uri=~s/^\///;
-    $uri = 'http://'.$hostname{ &homeserver($cnum,$cdom)}.'/raw/'.$uri;
+    $uri = 'http://'.&hostname(&homeserver($cnum,$cdom)).'/raw/'.$uri;
     my $ua=new LWP::UserAgent;
     my $request=new HTTP::Request($reqtype,$uri);
     my $response=$ua->request($request);
@@ -7114,15 +7550,16 @@ sub filelocation {
 	$file=~s-^/adm/wrapper/-/-;
 	$file=~s-^/adm/coursedocs/showdoc/-/-;
     }
+
     if ($file=~m:^/~:) { # is a contruction space reference
         $location = $file;
         $location =~ s:/~(.*?)/(.*):/home/$1/public_html/$2:;
-    } elsif ($file=~m:^/home/[^/]*/public_html/:) {
+    } elsif ($file=~m{^/home/$match_username/public_html/}) {
 	# is a correct contruction space reference
         $location = $file;
     } elsif ($file=~/^\/*(uploaded|editupload)/) { # is an uploaded file
         my ($udom,$uname,$filename)=
-  	    ($file=~m -^/+(?:uploaded|editupload)/+([^/]+)/+([^/]+)/+(.*)$-);
+  	    ($file=~m -^/+(?:uploaded|editupload)/+($match_domain)/+($match_name)/+(.*)$-);
         my $home=&homeserver($uname,$udom);
         my $is_me=0;
         my @ids=&current_machine_ids();
@@ -7134,6 +7571,8 @@ sub filelocation {
   	  $location=$Apache::lonnet::perlvar{'lonDocRoot'}.'/userfiles/'.
   	      $udom.'/'.$uname.'/'.$filename;
         }
+    } elsif ($file =~ m-^/adm/-) {
+	$location = $perlvar{'lonDocRoot'}.'/'.$file;
     } else {
         $file=~s/^\Q$perlvar{'lonDocRoot'}\E//;
         $file=~s:^/res/:/:;
@@ -7159,37 +7598,68 @@ sub hreflocation {
     }
     if ($file=~m-^\Q$perlvar{'lonDocRoot'}\E-) {
 	$file=~s-^\Q$perlvar{'lonDocRoot'}\E--;
-    } elsif ($file=~m-/home/(\w+)/public_html/-) {
-	$file=~s-^/home/(\w+)/public_html/-/~$1/-;
+    } elsif ($file=~m-/home/($match_username)/public_html/-) {
+	$file=~s-^/home/($match_username)/public_html/-/~$1/-;
     } elsif ($file=~m-^\Q$perlvar{'lonUsersDir'}\E-) {
-	$file=~s-^/home/httpd/lonUsers/([^/]*)/./././([^/]*)/userfiles/
+	$file=~s-^/home/httpd/lonUsers/($match_domain)/./././($match_name)/userfiles/
 	    -/uploaded/$1/$2/-x;
     }
     return $file;
 }
 
 sub current_machine_domains {
-    my $hostname=$hostname{$perlvar{'lonHostID'}};
+    return &machine_domains(&hostname($perlvar{'lonHostID'}));
+}
+
+sub machine_domains {
+    my ($hostname) = @_;
     my @domains;
+    my %hostname = &all_hostnames();
     while( my($id, $name) = each(%hostname)) {
 #	&logthis("-$id-$name-$hostname-");
 	if ($hostname eq $name) {
-	    push(@domains,$hostdom{$id});
+	    push(@domains,&host_domain($id));
 	}
     }
     return @domains;
 }
 
 sub current_machine_ids {
-    my $hostname=$hostname{$perlvar{'lonHostID'}};
+    return &machine_ids(&hostname($perlvar{'lonHostID'}));
+}
+
+sub machine_ids {
+    my ($hostname) = @_;
+    $hostname ||= &hostname($perlvar{'lonHostID'});
     my @ids;
-    while( my($id, $name) = each(%hostname)) {
-#	&logthis("-$id-$name-$hostname-");
-	if ($hostname eq $name) {
-	    push(@ids,$id);
-	}
+    my %name_to_host = &all_names();
+    if (ref($name_to_host{$hostname}) eq 'ARRAY') {
+	return @{ $name_to_host{$hostname} };
+    }
+    return;
+}
+
+sub additional_machine_domains {
+    my @domains;
+    open(my $fh,"<$perlvar{'lonTabDir'}/expected_domains.tab");
+    while( my $line = <$fh>) {
+        $line =~ s/\s//g;
+        push(@domains,$line);
+    }
+    return @domains;
+}
+
+sub default_login_domain {
+    my $domain = $perlvar{'lonDefDomain'};
+    my $testdomain=(split(/\./,$ENV{'HTTP_HOST'}))[0];
+    foreach my $posdom (&current_machine_domains(),
+                        &additional_machine_domains()) {
+        if (lc($posdom) eq lc($testdomain)) {
+            $domain=$posdom;
+            last;
+        }
     }
-    return @ids;
+    return $domain;
 }
 
 # ------------------------------------------------------------- Declutters URLs
@@ -7210,7 +7680,8 @@ sub declutter {
 
 sub clutter {
     my $thisfn='/'.&declutter(shift);
-    unless ($thisfn=~/^\/(uploaded|editupload|adm|userfiles|ext|raw|priv|public)\//) { 
+    if ($thisfn !~ m{^/(uploaded|editupload|adm|userfiles|ext|raw|priv|public)/}
+	|| $thisfn =~ m{^/adm/(includes|pages)} ) { 
        $thisfn='/res'.$thisfn; 
     }
     if ($thisfn !~m|/adm|) {
@@ -7279,104 +7750,331 @@ sub correct_line_ends {
 sub goodbye {
    &logthis("Starting Shut down");
 #not converted to using infrastruture and probably shouldn't be
-   &logthis(sprintf("%-20s is %s",'%badServerCache',length(&freeze(\%badServerCache))));
+   &logthis(sprintf("%-20s is %s",'%badServerCache',length(&nfreeze(\%badServerCache))));
 #converted
 #   &logthis(sprintf("%-20s is %s",'%metacache',scalar(%metacache)));
-   &logthis(sprintf("%-20s is %s",'%homecache',length(&freeze(\%homecache))));
-#   &logthis(sprintf("%-20s is %s",'%titlecache',length(&freeze(\%titlecache))));
-#   &logthis(sprintf("%-20s is %s",'%courseresdatacache',length(&freeze(\%courseresdatacache))));
+   &logthis(sprintf("%-20s is %s",'%homecache',length(&nfreeze(\%homecache))));
+#   &logthis(sprintf("%-20s is %s",'%titlecache',length(&nfreeze(\%titlecache))));
+#   &logthis(sprintf("%-20s is %s",'%courseresdatacache',length(&nfreeze(\%courseresdatacache))));
 #1.1 only
-#   &logthis(sprintf("%-20s is %s",'%userresdatacache',length(&freeze(\%userresdatacache))));
-#   &logthis(sprintf("%-20s is %s",'%getsectioncache',length(&freeze(\%getsectioncache))));
-#   &logthis(sprintf("%-20s is %s",'%courseresversioncache',length(&freeze(\%courseresversioncache))));
-#   &logthis(sprintf("%-20s is %s",'%resversioncache',length(&freeze(\%resversioncache))));
-   &logthis(sprintf("%-20s is %s",'%remembered',length(&freeze(\%remembered))));
+#   &logthis(sprintf("%-20s is %s",'%userresdatacache',length(&nfreeze(\%userresdatacache))));
+#   &logthis(sprintf("%-20s is %s",'%getsectioncache',length(&nfreeze(\%getsectioncache))));
+#   &logthis(sprintf("%-20s is %s",'%courseresversioncache',length(&nfreeze(\%courseresversioncache))));
+#   &logthis(sprintf("%-20s is %s",'%resversioncache',length(&nfreeze(\%resversioncache))));
+   &logthis(sprintf("%-20s is %s",'%remembered',length(&nfreeze(\%remembered))));
    &logthis(sprintf("%-20s is %s",'kicks',$kicks));
    &logthis(sprintf("%-20s is %s",'hits',$hits));
    &flushcourselogs();
    &logthis("Shutting down");
 }
 
-BEGIN {
-# ----------------------------------- Read loncapa.conf and loncapa_apache.conf
-    unless ($readit) {
-{
-    my $configvars = LONCAPA::Configuration::read_conf('loncapa.conf');
-    %perlvar = (%perlvar,%{$configvars});
-}
+sub get_dns {
+    my ($url,$func,$ignore_cache) = @_;
+    if (!$ignore_cache) {
+	my ($content,$cached)=
+	    &Apache::lonnet::is_cached_new('dns',$url);
+	if ($cached) {
+	    &$func($content);
+	    return;
+	}
+    }
 
+    my %alldns;
+    open(my $config,"<$perlvar{'lonTabDir'}/hosts.tab");
+    foreach my $dns (<$config>) {
+	next if ($dns !~ /^\^(\S*)/x);
+	$alldns{$1} = 1;
+    }
+    while (%alldns) {
+	my ($dns) = keys(%alldns);
+	delete($alldns{$dns});
+	my $ua=new LWP::UserAgent;
+	my $request=new HTTP::Request('GET',"http://$dns$url");
+	my $response=$ua->request($request);
+	next if ($response->is_error());
+	my @content = split("\n",$response->content);
+	&Apache::lonnet::do_cache_new('dns',$url,\@content,30*24*60*60);
+	&$func(\@content);
+	return;
+    }
+    close($config);
+    my $which = (split('/',$url))[3];
+    &logthis("unable to contact DNS defaulting to on disk file dns_$which.tab\n");
+    open($config,"<$perlvar{'lonTabDir'}/dns_$which.tab");
+    my @content = <$config>;
+    &$func(\@content);
+    return;
+}
 # ------------------------------------------------------------ Read domain file
 {
-    %domaindescription = ();
-    %domain_auth_def = ();
-    %domain_auth_arg_def = ();
-    my $fh;
-    if (open($fh,"<".$Apache::lonnet::perlvar{'lonTabDir'}.'/domain.tab')) {
-	while (my $line = <$fh>) {
-           next if ($line =~ /^(\#|\s*$)/);
-#           next if /^\#/;
-           chomp $line;
-           my ($domain, $domain_description, $def_auth, $def_auth_arg,
-	       $def_lang, $city, $longi, $lati, $primary) = split(/:/,$line,9);
-	   $domain_auth_def{$domain}=$def_auth;
-           $domain_auth_arg_def{$domain}=$def_auth_arg;
-	   $domaindescription{$domain}=$domain_description;
-	   $domain_lang_def{$domain}=$def_lang;
-	   $domain_city{$domain}=$city;
-	   $domain_longi{$domain}=$longi;
-	   $domain_lati{$domain}=$lati;
-           $domain_primary{$domain}=$primary;
+    my $loaded;
+    my %domain;
+
+    sub parse_domain_tab {
+	my ($lines) = @_;
+	foreach my $line (@$lines) {
+	    next if ($line =~ /^(\#|\s*$ )/x);
+
+	    chomp($line);
+	    my ($name,@elements) = split(/:/,$line,9);
+	    my %this_domain;
+	    foreach my $field ('description', 'auth_def', 'auth_arg_def',
+			       'lang_def', 'city', 'longi', 'lati',
+			       'primary') {
+		$this_domain{$field} = shift(@elements);
+	    }
+	    $domain{$name} = \%this_domain;
+	}
+    }
+
+    sub reset_domain_info {
+	undef($loaded);
+	undef(%domain);
+    }
+
+    sub load_domain_tab {
+	my ($ignore_cache) = @_;
+	&get_dns('/adm/dns/domain',\&parse_domain_tab,$ignore_cache);
+	my $fh;
+	if (open($fh,"<".$perlvar{'lonTabDir'}.'/domain.tab')) {
+	    my @lines = <$fh>;
+	    &parse_domain_tab(\@lines);
+	}
+	close($fh);
+	$loaded = 1;
+    }
+
+    sub domain {
+	&load_domain_tab() if (!$loaded);
+
+	my ($name,$what) = @_;
+	return if ( !exists($domain{$name}) );
 
- #         &logthis("Domain.tab: $domain, $domain_auth_def{$domain}, $domain_auth_arg_def{$domain},$domaindescription{$domain}");
-#          &logthis("Domain.tab: $domain ".$domaindescription{$domain} );
+	if (!$what) {
+	    return $domain{$name}{'description'};
 	}
+	return $domain{$name}{$what};
     }
-    close ($fh);
 }
 
 
 # ------------------------------------------------------------- Read hosts file
 {
-    open(my $config,"<$perlvar{'lonTabDir'}/hosts.tab");
+    my %hostname;
+    my %hostdom;
+    my %libserv;
+    my $loaded;
+    my %name_to_host;
+
+    sub parse_hosts_tab {
+	my ($file) = @_;
+	foreach my $configline (@$file) {
+	    next if ($configline =~ /^(\#|\s*$ )/x);
+	    next if ($configline =~ /^\^/);
+	    chomp($configline);
+	    my ($id,$domain,$role,$name)=split(/:/,$configline);
+	    $name=~s/\s//g;
+	    if ($id && $domain && $role && $name) {
+		$hostname{$id}=$name;
+		push(@{$name_to_host{$name}}, $id);
+		$hostdom{$id}=$domain;
+		if ($role eq 'library') { $libserv{$id}=$name; }
+	    }
+	}
+    }
+    
+    sub reset_hosts_info {
+	&reset_domain_info();
+	&reset_hosts_ip_info();
+	undef(%name_to_host);
+	undef(%hostname);
+	undef(%hostdom);
+	undef(%libserv);
+	undef($loaded);
+    }
 
-    while (my $configline=<$config>) {
-       next if ($configline =~ /^(\#|\s*$)/);
-       chomp($configline);
-       my ($id,$domain,$role,$name)=split(/:/,$configline);
-       $name=~s/\s//g;
-       if ($id && $domain && $role && $name) {
-	 $hostname{$id}=$name;
-	 $hostdom{$id}=$domain;
-	 if ($role eq 'library') { $libserv{$id}=$name; }
-       }
+    sub load_hosts_tab {
+	my ($ignore_cache) = @_;
+	&get_dns('/adm/dns/hosts',\&parse_hosts_tab,$ignore_cache);
+	open(my $config,"<$perlvar{'lonTabDir'}/hosts.tab");
+	my @config = <$config>;
+	&parse_hosts_tab(\@config);
+	close($config);
+	$loaded=1;
+    }
+
+    sub hostname {
+	&load_hosts_tab() if (!$loaded);
+
+	my ($lonid) = @_;
+	return $hostname{$lonid};
+    }
+
+    sub all_hostnames {
+	&load_hosts_tab() if (!$loaded);
+
+	return %hostname;
+    }
+
+    sub all_names {
+	&load_hosts_tab() if (!$loaded);
+
+	return %name_to_host;
+    }
+
+    sub is_library {
+	&load_hosts_tab() if (!$loaded);
+
+	return exists($libserv{$_[0]});
+    }
+
+    sub all_library {
+	&load_hosts_tab() if (!$loaded);
+
+	return %libserv;
+    }
+
+    sub get_servers {
+	&load_hosts_tab() if (!$loaded);
+
+	my ($domain,$type) = @_;
+	my %possible_hosts = ($type eq 'library') ? %libserv
+	                                          : %hostname;
+	my %result;
+	if (ref($domain) eq 'ARRAY') {
+	    while ( my ($host,$hostname) = each(%possible_hosts)) {
+		if (grep(/^\Q$hostdom{$host}\E$/,@$domain)) {
+		    $result{$host} = $hostname;
+		}
+	    }
+	} else {
+	    while ( my ($host,$hostname) = each(%possible_hosts)) {
+		if ($hostdom{$host} eq $domain) {
+		    $result{$host} = $hostname;
+		}
+	    }
+	}
+	return %result;
+    }
+
+    sub host_domain {
+	&load_hosts_tab() if (!$loaded);
+
+	my ($lonid) = @_;
+	return $hostdom{$lonid};
+    }
+
+    sub all_domains {
+	&load_hosts_tab() if (!$loaded);
+
+	my %seen;
+	my @uniq = grep(!$seen{$_}++, values(%hostdom));
+	return @uniq;
     }
-    close($config);
-    # FIXME: dev server don't want this, production servers _do_ want this
-    #&get_iphost();
 }
 
-sub get_iphost {
-    if (%iphost) { return %iphost; }
+{ 
+    my %iphost;
     my %name_to_ip;
-    foreach my $id (keys(%hostname)) {
-	my $name=$hostname{$id};
-	my $ip;
-	if (!exists($name_to_ip{$name})) {
-	    $ip = gethostbyname($name);
-	    if (!$ip || length($ip) ne 4) {
-		&logthis("Skipping host $id name $name no IP found\n");
-		next;
+    my %lonid_to_ip;
+
+    sub get_hosts_from_ip {
+	my ($ip) = @_;
+	my %iphosts = &get_iphost();
+	if (ref($iphosts{$ip})) {
+	    return @{$iphosts{$ip}};
+	}
+	return;
+    }
+    
+    sub reset_hosts_ip_info {
+	undef(%iphost);
+	undef(%name_to_ip);
+	undef(%lonid_to_ip);
+    }
+
+    sub get_host_ip {
+	my ($lonid) = @_;
+	if (exists($lonid_to_ip{$lonid})) {
+	    return $lonid_to_ip{$lonid};
+	}
+	my $name=&hostname($lonid);
+   	my $ip = gethostbyname($name);
+	return if (!$ip || length($ip) ne 4);
+	$ip=inet_ntoa($ip);
+	$name_to_ip{$name}   = $ip;
+	$lonid_to_ip{$lonid} = $ip;
+	return $ip;
+    }
+    
+    sub get_iphost {
+	my ($ignore_cache) = @_;
+
+	if (!$ignore_cache) {
+	    if (%iphost) {
+		return %iphost;
+	    }
+	    my ($ip_info,$cached)=
+		&Apache::lonnet::is_cached_new('iphost','iphost');
+	    if ($cached) {
+		%iphost      = %{$ip_info->[0]};
+		%name_to_ip  = %{$ip_info->[1]};
+		%lonid_to_ip = %{$ip_info->[2]};
+		return %iphost;
 	    }
-	    $ip=inet_ntoa($ip);
-	    $name_to_ip{$name} = $ip;
-	} else {
-	    $ip = $name_to_ip{$name};
 	}
-	push(@{$iphost{$ip}},$id);
+
+	# get yesterday's info for fallback
+	my %old_name_to_ip;
+	my ($ip_info,$cached)=
+	    &Apache::lonnet::is_cached_new('iphost','iphost');
+	if ($cached) {
+	    %old_name_to_ip = %{$ip_info->[1]};
+	}
+
+	my %name_to_host = &all_names();
+	foreach my $name (keys(%name_to_host)) {
+	    my $ip;
+	    if (!exists($name_to_ip{$name})) {
+		$ip = gethostbyname($name);
+		if (!$ip || length($ip) ne 4) {
+		    if (defined($old_name_to_ip{$name})) {
+			$ip = $old_name_to_ip{$name};
+			&logthis("Can't find $name defaulting to old $ip");
+		    } else {
+			&logthis("Name $name no IP found");
+			next;
+		    }
+		} else {
+		    $ip=inet_ntoa($ip);
+		}
+		$name_to_ip{$name} = $ip;
+	    } else {
+		$ip = $name_to_ip{$name};
+	    }
+	    foreach my $id (@{ $name_to_host{$name} }) {
+		$lonid_to_ip{$id} = $ip;
+	    }
+	    push(@{$iphost{$ip}},@{$name_to_host{$name}});
+	}
+	&Apache::lonnet::do_cache_new('iphost','iphost',
+				      [\%iphost,\%name_to_ip,\%lonid_to_ip],
+				      48*60*60);
+
+	return %iphost;
     }
-    return %iphost;
 }
 
+BEGIN {
+
+# ----------------------------------- Read loncapa.conf and loncapa_apache.conf
+    unless ($readit) {
+{
+    my $configvars = LONCAPA::Configuration::read_conf('loncapa.conf');
+    %perlvar = (%perlvar,%{$configvars});
+}
+
+
 # ------------------------------------------------------ Read spare server file
 {
     open(my $config,"<$perlvar{'lonTabDir'}/spare.tab");
@@ -7708,6 +8406,16 @@ X<userenvironment()>
 B<userenvironment($udom,$uname,@what)>: gets the values of the keys
 passed in @what from the requested user's environment, returns a hash
 
+=item * 
+X<userlog_query()>
+B<userlog_query($uname,$udom,%filters)>: retrieves data from a user's
+activity.log file. %filters defines filters applied when parsing the
+log file. These can be start or end timestamps, or the type of action
+- log to look for Login or Logout events, check for Checkin or
+Checkout, role for role selection. The response is in the form
+timestamp1:hostid1:event1&timestamp2:hostid2:event2 where events are
+escaped strings of the action recorded in the activity.log file.
+
 =back
 
 =head2 User Roles
@@ -7716,8 +8424,7 @@ passed in @what from the requested user'
 
 =item *
 
-allowed($priv,$uri) : check for a user privilege; returns codes for allowed
-actions
+allowed($priv,$uri,$symb,$role) : check for a user privilege; returns codes for allowed actions
  F: full access
  U,I,K: authentication modes (cxx only)
  '': forbidden
@@ -7736,6 +8443,21 @@ and course level
 plaintext($short) : return value in %prp hash (rolesplain.tab); plain text
 explanation of a user role term
 
+=item *
+
+get_my_roles($uname,$udom,$context,$types,$roles,$roledoms) :
+All arguments are optional. Returns a hash of a roles, either for
+co-author/assistant author roles for a user's Construction Space
+(default), or if $context is 'user', roles for the user himself,
+In the hash, keys are set to colon-sparated $uname,$udom,and $role,
+and value is set to colon-separated start and end times for the role.
+If no username and domain are specified, will default to current
+user/domain. Types, roles, and roledoms are references to arrays,
+of role statuses (active, future or previous), roles 
+(e.g., cc,in, st etc.) and domains of the roles which can be used
+to restrict the list of roles reported. If no array ref is 
+provided for types, will default to return only active roles.
+
 =back
 
 =head2 User Modification
@@ -7878,6 +8600,14 @@ setting for a specific $type, where $typ
 @what should be a list of parameters to ask about. This routine caches
 answers for 5 minutes.
 
+=item *
+
+get_courseresdata($courseid, $domain) : dump the entire course resource
+data base, returning a hash that is keyed by the resource name and has
+values that are the resource value.  I believe that the timestamps and
+versions are also returned.
+
+
 =back
 
 =head2 Course Modification
@@ -8159,12 +8889,15 @@ critical subroutine
 
 =item *
 
-get_dom($namespace,$storearr,$udomain) : returns hash with keys from array
-reference filled in from namespace found in domain level on primary domain server ($udomain is optional)
+get_dom($namespace,$storearr,$udom,$uhome) : returns hash with keys from
+array reference filled in from namespace found in domain level on either
+specified domain server ($uhome) or primary domain server ($udom and $uhome are optional).
 
 =item *
 
-put_dom($namespace,$storehash,$udomain) :  stores hash in namespace at domain level on primary domain server ($udomain is optional)
+put_dom($namespace,$storehash,$udom,$uhome) :  stores hash in namespace at 
+domain level either on specified domain server ($uhome) or primary domain 
+server ($udom and $uhome are optional)
 
 =back
 
@@ -8557,3 +9290,4 @@ symblist($mapname,%newhash) : update sym
 =back
 
 =cut
+