--- loncom/lonnet/perl/lonnet.pm	2012/03/14 20:53:50	1.1158
+++ loncom/lonnet/perl/lonnet.pm	2017/08/07 20:22:54	1.1347
@@ -1,7 +1,7 @@
 # The LearningOnline Network
 # TCP networking package
 #
-# $Id: lonnet.pm,v 1.1158 2012/03/14 20:53:50 www Exp $
+# $Id: lonnet.pm,v 1.1347 2017/08/07 20:22:54 raeburn Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -71,10 +71,12 @@ delayed.
 package Apache::lonnet;
 
 use strict;
-use LWP::UserAgent();
 use HTTP::Date;
 use Image::Magick;
 
+
+use Encode;
+
 use vars qw(%perlvar %spareid %pr %prp $memcache %packagetab $tmpdir
             $_64bit %env %protocol %loncaparevs %serverhomeIDs %needsrelease
             %managerstab);
@@ -89,18 +91,21 @@ use GDBM_File;
 use HTML::LCParser;
 use Fcntl qw(:flock);
 use Storable qw(thaw nfreeze);
-use Time::HiRes qw( gettimeofday tv_interval );
+use Time::HiRes qw( sleep gettimeofday tv_interval );
 use Cache::Memcached;
 use Digest::MD5;
 use Math::Random;
 use File::MMagic;
 use LONCAPA qw(:DEFAULT :match);
 use LONCAPA::Configuration;
+use LONCAPA::lonmetadata;
+use LONCAPA::Lond;
+use LONCAPA::LWPReq;
 
 use File::Copy;
 
 my $readit;
-my $max_connection_retries = 10;     # Or some such value.
+my $max_connection_retries = 20;     # Or some such value.
 
 require Exporter;
 
@@ -108,30 +113,33 @@ our @ISA = qw (Exporter);
 our @EXPORT = qw(%env);
 
 
-# --------------------------------------------------------------------- Logging
+# ------------------------------------ Logging (parameters, docs, slots, roles)
 {
     my $logid;
-    sub instructor_log {
-	my ($hash_name,$storehash,$delflag,$uname,$udom,$cnum,$cdom)=@_;
-        if (($cnum eq '') || ($cdom eq '')) {
-            $cnum = $env{'course.'.$env{'request.course.id'}.'.num'};
-            $cdom = $env{'course.'.$env{'request.course.id'}.'.domain'};
+    sub write_log {
+	my ($context,$hash_name,$storehash,$delflag,$uname,$udom,$cnum,$cdom)=@_;
+        if ($context eq 'course') {
+            if (($cnum eq '') || ($cdom eq '')) {
+                $cnum = $env{'course.'.$env{'request.course.id'}.'.num'};
+                $cdom = $env{'course.'.$env{'request.course.id'}.'.domain'};
+            }
         }
-	$logid++;
+	$logid ++;
         my $now = time();
 	my $id=$now.'00000'.$$.'00000'.$logid;
-	return &Apache::lonnet::put('nohist_'.$hash_name,
-				    { $id => {
-					'exe_uname' => $env{'user.name'},
-					'exe_udom'  => $env{'user.domain'},
-					'exe_time'  => $now,
-					'exe_ip'    => $ENV{'REMOTE_ADDR'},
-					'delflag'   => $delflag,
-					'logentry'  => $storehash,
-					'uname'     => $uname,
-					'udom'      => $udom,
-				    }
-				  },$cdom,$cnum);
+        my $logentry = { 
+                          $id => {
+                                   'exe_uname' => $env{'user.name'},
+                                   'exe_udom'  => $env{'user.domain'},
+                                   'exe_time'  => $now,
+                                   'exe_ip'    => $ENV{'REMOTE_ADDR'},
+                                   'delflag'   => $delflag,
+                                   'logentry'  => $storehash,
+                                   'uname'     => $uname,
+                                   'udom'      => $udom,
+                                  }
+                       };
+	return &put('nohist_'.$hash_name,$logentry,$cdom,$cnum);
     }
 }
 
@@ -221,6 +229,52 @@ sub get_server_distarch {
     return;
 }
 
+sub get_servercerts_info {
+    my ($lonhost,$context) = @_;
+    my ($rep,$uselocal);
+    if (grep { $_ eq $lonhost } &current_machine_ids()) {
+        $uselocal = 1;
+    }
+    if (($context ne 'cgi') && ($uselocal)) {
+        my $distro = (split(/\:/,&get_server_distarch($lonhost)))[0];
+        if ($distro eq '') {
+            $uselocal = 0;
+        } elsif ($distro =~ /^(?:centos|redhat|scientific)(\d+)$/) {
+            if ($1 < 6) {
+                $uselocal = 0;
+            }
+        }  elsif ($distro =~ /^(?:sles)(\d+)$/) {
+            if ($1 < 12) {
+                $uselocal = 0;
+            }
+        }
+    }
+    if ($uselocal) {
+        $rep = LONCAPA::Lond::server_certs(\%perlvar);
+    } else {
+        $rep=&reply('servercerts',$lonhost);
+    }
+    my ($result,%returnhash);
+    if (defined($lonhost)) {
+        if (!defined(&hostname($lonhost))) {
+            return;
+        }
+    }
+    if (($rep=~/^(refused|rejected|error)/) || ($rep eq 'con_lost') ||
+        ($rep eq 'unknown_cmd')) {
+        $result = $rep;
+    } else {
+        $result = 'ok';
+        my @pairs=split(/\&/,$rep);
+        foreach my $item (@pairs) {
+            my ($key,$value)=split(/=/,$item,2);
+            my $what = &unescape($key);
+            $returnhash{$what}=&thaw_unescape($value);
+        }
+    }
+    return ($result,\%returnhash);
+}
+
 sub get_server_loncaparev {
     my ($dom,$lonhost,$ignore_cache,$caller) = @_;
     if (defined($lonhost)) {
@@ -255,13 +309,11 @@ sub get_server_loncaparev {
             $answer = &reply('serverloncaparev',$lonhost);
             if (($answer eq 'unknown_cmd') || ($answer eq 'con_lost')) {
                 if ($caller eq 'loncron') {
-                    my $ua=new LWP::UserAgent;
-                    $ua->timeout(4);
                     my $protocol = $protocol{$lonhost};
                     $protocol = 'http' if ($protocol ne 'https');
                     my $url = $protocol.'://'.&hostname($lonhost).'/adm/about.html';
                     my $request=new HTTP::Request('GET',$url);
-                    my $response=$ua->request($request);
+                    my $response=&LONCAPA::LWPReq::makerequest($lonhost,$request,'',\%perlvar,4,1);
                     unless ($response->is_error()) {
                         my $content = $response->content;
                         if ($content =~ /<p>VERSION\:\s*([\w.\-]+)<\/p>/) {
@@ -348,8 +400,11 @@ sub get_remote_globals {
 }
 
 sub remote_devalidate_cache {
-    my ($lonhost,$name,$id) = @_;
-    my $response = &reply('devalidatecache:'.&escape($name).':'.&escape($id),$lonhost);
+    my ($lonhost,$cachekeys) = @_;
+    my $items;
+    return unless (ref($cachekeys) eq 'ARRAY');
+    my $cachestr = join('&',@{$cachekeys});
+    my $response = &reply('devalidatecache:'.&escape($cachestr),$lonhost);
     return $response;
 }
 
@@ -364,7 +419,7 @@ sub subreply {
 
     my $lockfile=$peerfile.".lock";
     while (-e $lockfile) {	# Need to wait for the lockfile to disappear.
-	sleep(1);
+	sleep(0.1);
     }
     # At this point, either a loncnew parent is listening or an old lonc
     # or loncnew child is listening so we can connect or everything's dead.
@@ -382,7 +437,7 @@ sub subreply {
 	} else {
 	    &create_connection(&hostname($server),$server);
 	}
-        sleep(1);		# Try again later if failed connection.
+        sleep(0.1);	# Try again later if failed connection.
     }
     my $answer;
     if ($client) {
@@ -411,8 +466,8 @@ sub reply {
 
 sub reconlonc {
     my ($lonid) = @_;
-    my $hostname = &hostname($lonid);
     if ($lonid) {
+        my $hostname = &hostname($lonid);
 	my $peerfile="$perlvar{'lonSockDir'}/$hostname";
 	if ($hostname && -e $peerfile) {
 	    &logthis("Trying to reconnect lonc for $lonid ($hostname)");
@@ -437,7 +492,7 @@ sub reconlonc {
 	    &logthis("lonc at pid $loncpid responding, sending USR1");
             kill USR1 => $loncpid;
             sleep 1;
-         } else {
+        } else {
 	    &logthis(
                "<font color=\"blue\">WARNING:".
                " lonc at pid $loncpid not responding, giving up</font>");
@@ -458,7 +513,7 @@ sub critical {
     }
     my $answer=reply($cmd,$server);
     if ($answer eq 'con_lost') {
-	&reconlonc("$perlvar{'lonSockDir'}/$server");
+	&reconlonc($server);
 	my $answer=reply($cmd,$server);
         if ($answer eq 'con_lost') {
             my $now=time;
@@ -475,7 +530,7 @@ sub critical {
 		    close($dfh);
 		}
             }
-            sleep 2;
+            sleep 1;
             my $wcmd='';
             {
 		my $dfh;
@@ -595,12 +650,25 @@ sub transfer_profile_to_env {
 
 # ---------------------------------------------------- Check for valid session 
 sub check_for_valid_session {
-    my ($r,$name) = @_;
+    my ($r,$name,$userhashref) = @_;
     my %cookies=CGI::Cookie->parse($r->header_in('Cookie'));
+    my ($linkname,$pubname);
     if ($name eq '') {
         $name = 'lonID';
+        $linkname = 'lonLinkID';
+        $pubname = 'lonPubID';
     }
     my $lonid=$cookies{$name};
+    if (!$lonid) {
+        if (($name eq 'lonID') && ($ENV{'SERVER_PORT'} != 443) && ($linkname)) {
+            $lonid=$cookies{$linkname};
+        }
+        if (!$lonid) {
+            if (($name eq 'lonID') && ($pubname)) {
+                $lonid=$cookies{$pubname};
+            }
+        }
+    }
     return undef if (!$lonid);
 
     my $handle=&LONCAPA::clean_handle($lonid->value);
@@ -626,6 +694,12 @@ sub check_for_valid_session {
 	|| !defined($disk_env{'user.domain'})) {
 	return undef;
     }
+
+    if (ref($userhashref) eq 'HASH') {
+        $userhashref->{'name'} = $disk_env{'user.name'};
+        $userhashref->{'domain'} = $disk_env{'user.domain'};
+    }
+
     return $handle;
 }
 
@@ -659,7 +733,7 @@ sub appenv {
 	    if (($key =~ /^user\.role/) || ($key =~ /^user\.priv/)) {
                 $refused = 1;
                 if (ref($roles) eq 'ARRAY') {
-                    my ($type,$role) = ($key =~ /^user\.(role|priv)\.([^.]+)\./);
+                    my ($type,$role) = ($key =~ m{^user\.(role|priv)\.(.+?)\./});
                     if (grep(/^\Q$role\E$/,@{$roles})) {
                         $refused = 0;
                     }
@@ -832,10 +906,8 @@ sub spareserver {
     if (ref($spareshash) eq 'HASH') {
         if (ref($spareshash->{'primary'}) eq 'ARRAY') {
             foreach my $try_server (@{ $spareshash->{'primary'} }) {
-                if ($uint_dom) {
-                    next unless (&spare_can_host($udom,$uint_dom,$remotesessions,
-                                                 $try_server));
-                }
+                next unless (&spare_can_host($udom,$uint_dom,$remotesessions,
+                                             $try_server));
 	        ($spare_server, $lowest_load) =
 	            &compare_server_load($try_server, $spare_server, $lowest_load);
             }
@@ -846,10 +918,8 @@ sub spareserver {
         if (!$found_server) {
             if (ref($spareshash->{'default'}) eq 'ARRAY') { 
 	        foreach my $try_server (@{ $spareshash->{'default'} }) {
-                    if ($uint_dom) {
-                        next unless (&spare_can_host($udom,$uint_dom,
-                                                     $remotesessions,$try_server));
-                    }
+                    next unless (&spare_can_host($udom,$uint_dom,
+                                                 $remotesessions,$try_server));
 	            ($spare_server, $lowest_load) =
 		        &compare_server_load($try_server, $spare_server, $lowest_load);
                 }
@@ -873,7 +943,17 @@ sub spareserver {
 }
 
 sub compare_server_load {
-    my ($try_server, $spare_server, $lowest_load) = @_;
+    my ($try_server, $spare_server, $lowest_load, $required) = @_;
+
+    if ($required) {
+        my ($reqdmajor,$reqdminor) = ($required =~ /^(\d+)\.(\d+)$/);
+        my $remoterev = &get_server_loncaparev(undef,$try_server);
+        my ($major,$minor) = ($remoterev =~ /^\'?(\d+)\.(\d+)\.[\w.\-]+\'?$/);
+        if (($major eq '' && $minor eq '') ||
+            (($reqdmajor > $major) || (($reqdmajor == $major) && ($reqdminor > $minor)))) {
+            return ($spare_server,$lowest_load);
+        }
+    }
 
     my $loadans     = &reply('load',    $try_server);
     my $userloadans = &reply('userload',$try_server);
@@ -934,26 +1014,43 @@ sub has_user_session {
 # --------- determine least loaded server in a user's domain which allows login
 
 sub choose_server {
-    my ($udom,$checkloginvia) = @_;
+    my ($udom,$checkloginvia,$required,$skiploadbal) = @_;
     my %domconfhash = &Apache::loncommon::get_domainconf($udom);
     my %servers = &get_servers($udom);
     my $lowest_load = 30000;
-    my ($login_host,$hostname,$portal_path,$isredirect);
+    my ($login_host,$hostname,$portal_path,$isredirect,$balancers);
+    if ($skiploadbal) {
+        ($balancers,my $cached)=&is_cached_new('loadbalancing',$udom);
+        unless (defined($cached)) {
+            my $cachetime = 60*60*24;
+            my %domconfig =
+                &Apache::lonnet::get_dom('configuration',['loadbalancing'],$udom);
+            if (ref($domconfig{'loadbalancing'}) eq 'HASH') {
+                $balancers = &do_cache_new('loadbalancing',$udom,$domconfig{'loadbalancing'},
+                                           $cachetime);
+            }
+        }
+    }
     foreach my $lonhost (keys(%servers)) {
+        if ($skiploadbal) {
+            if (ref($balancers) eq 'HASH') {
+                next if (exists($balancers->{$lonhost}));
+            }
+        }   
         my $loginvia;
         if ($checkloginvia) {
             $loginvia = $domconfhash{$udom.'.login.loginvia_'.$lonhost};
             if ($loginvia) {
                 my ($server,$path) = split(/:/,$loginvia);
                 ($login_host, $lowest_load) =
-                    &compare_server_load($server, $login_host, $lowest_load);
+                    &compare_server_load($server, $login_host, $lowest_load, $required);
                 if ($login_host eq $server) {
                     $portal_path = $path;
                     $isredirect = 1;
                 }
             } else {
                 ($login_host, $lowest_load) =
-                    &compare_server_load($lonhost, $login_host, $lowest_load);
+                    &compare_server_load($lonhost, $login_host, $lowest_load, $required);
                 if ($login_host eq $lonhost) {
                     $portal_path = '';
                     $isredirect = ''; 
@@ -961,13 +1058,13 @@ sub choose_server {
             }
         } else {
             ($login_host, $lowest_load) =
-                &compare_server_load($lonhost, $login_host, $lowest_load);
+                &compare_server_load($lonhost, $login_host, $lowest_load, $required);
         }
     }
     if ($login_host ne '') {
         $hostname = &hostname($login_host);
     }
-    return ($login_host,$hostname,$portal_path,$isredirect);
+    return ($login_host,$hostname,$portal_path,$isredirect,$lowest_load);
 }
 
 # --------------------------------------------- Try to change a user's password
@@ -1134,19 +1231,27 @@ sub can_host_session {
 sub spare_can_host {
     my ($udom,$uint_dom,$remotesessions,$try_server)=@_;
     my $canhost=1;
-    my @intdoms;
-    my $internet_names = &Apache::lonnet::get_internet_names($try_server);
-    if (ref($internet_names) eq 'ARRAY') {
-        @intdoms = @{$internet_names};
-    }
-    unless (grep(/^\Q$uint_dom\E$/,@intdoms)) {
-        my $serverhomeID = &Apache::lonnet::get_server_homeID($try_server);
-        my $serverhomedom = &Apache::lonnet::host_domain($serverhomeID);
-        my %defdomdefaults = &Apache::lonnet::get_domain_defaults($serverhomedom);
-        my $remoterev = &Apache::lonnet::get_server_loncaparev(undef,$try_server);
-        $canhost = &can_host_session($udom,$try_server,$remoterev,
-                                     $remotesessions,
-                                     $defdomdefaults{'hostedsessions'});
+    my $try_server_hostname = &hostname($try_server);
+    my $serverhomeID = &get_server_homeID($try_server_hostname);
+    my $serverhomedom = &host_domain($serverhomeID);
+    my %defdomdefaults = &get_domain_defaults($serverhomedom);
+    if (ref($defdomdefaults{'offloadnow'}) eq 'HASH') {
+        if ($defdomdefaults{'offloadnow'}{$try_server}) {
+            $canhost = 0;
+        }
+    }
+    if (($canhost) && ($uint_dom)) {
+        my @intdoms;
+        my $internet_names = &get_internet_names($try_server);
+        if (ref($internet_names) eq 'ARRAY') {
+            @intdoms = @{$internet_names};
+        }
+        unless (grep(/^\Q$uint_dom\E$/,@intdoms)) {
+            my $remoterev = &get_server_loncaparev(undef,$try_server);
+            $canhost = &can_host_session($udom,$try_server,$remoterev,
+                                         $remotesessions,
+                                         $defdomdefaults{'hostedsessions'});
+        }
     }
     return $canhost;
 }
@@ -1231,15 +1336,16 @@ sub get_lonbalancer_config {
 }
 
 sub check_loadbalancing {
-    my ($uname,$udom) = @_;
-    my ($is_balancer,$dom_in_use,$homeintdom,$rule_in_effect,
-        $offloadto,$otherserver);
+    my ($uname,$udom,$caller) = @_;
+    my ($is_balancer,$currtargets,$currrules,$dom_in_use,$homeintdom,
+        $rule_in_effect,$offloadto,$otherserver);
     my $lonhost = $perlvar{'lonHostID'};
+    my @hosts = &current_machine_ids();
     my $uprimary_id = &Apache::lonnet::domain($udom,'primary');
     my $uintdom = &Apache::lonnet::internet_dom($uprimary_id);
     my $intdom = &Apache::lonnet::internet_dom($lonhost);
     my $serverhomedom = &host_domain($lonhost);
-
+    my $domneedscache;
     my $cachetime = 60*60*24;
 
     if (($uintdom ne '') && ($uintdom eq $intdom)) {
@@ -1254,18 +1360,13 @@ sub check_loadbalancing {
             &Apache::lonnet::get_dom('configuration',['loadbalancing'],$dom_in_use);
         if (ref($domconfig{'loadbalancing'}) eq 'HASH') {
             $result = &do_cache_new('loadbalancing',$dom_in_use,$domconfig{'loadbalancing'},$cachetime);
+        } else {
+            $domneedscache = $dom_in_use;
         }
     }
     if (ref($result) eq 'HASH') {
-        my $currbalancer = $result->{'lonhost'};
-        my $currtargets = $result->{'targets'};
-        my $currrules = $result->{'rules'};
-        if ($currbalancer ne '') {
-            my @hosts = &current_machine_ids();
-            if (grep(/^\Q$currbalancer\E$/,@hosts)) {
-                $is_balancer = 1;
-            }
-        }
+        ($is_balancer,$currtargets,$currrules) = 
+            &check_balancer_result($result,@hosts);
         if ($is_balancer) {
             if (ref($currrules) eq 'HASH') {
                 if ($homeintdom) {
@@ -1314,21 +1415,20 @@ sub check_loadbalancing {
             }
         }
     } elsif (($homeintdom) && ($udom ne $serverhomedom)) {
-        my ($result,$cached)=&is_cached_new('loadbalancing',$serverhomedom);
+        ($result,$cached)=&is_cached_new('loadbalancing',$serverhomedom);
         unless (defined($cached)) {
             my %domconfig =
                 &Apache::lonnet::get_dom('configuration',['loadbalancing'],$serverhomedom);
             if (ref($domconfig{'loadbalancing'}) eq 'HASH') {
-                $result = &do_cache_new('loadbalancing',$dom_in_use,$domconfig{'loadbalancing'},$cachetime);
+                $result = &do_cache_new('loadbalancing',$serverhomedom,$domconfig{'loadbalancing'},$cachetime);
+            } else {
+                $domneedscache = $serverhomedom;
             }
         }
         if (ref($result) eq 'HASH') {
-            my $currbalancer = $result->{'lonhost'};
-            my $currtargets = $result->{'targets'};
-            my $currrules = $result->{'rules'};
-
-            if ($currbalancer eq $lonhost) {
-                $is_balancer = 1;
+            ($is_balancer,$currtargets,$currrules) = 
+                &check_balancer_result($result,@hosts);
+            if ($is_balancer) {
                 if (ref($currrules) eq 'HASH') {
                     if ($currrules->{'_LC_internetdom'} ne '') {
                         $rule_in_effect = $currrules->{'_LC_internetdom'};
@@ -1342,48 +1442,99 @@ sub check_loadbalancing {
                 $is_balancer = 1;
                 $offloadto = &this_host_spares($dom_in_use);
             }
+            unless (defined($cached)) {
+                $domneedscache = $serverhomedom;
+            }
         }
     } else {
         if ($perlvar{'lonBalancer'} eq 'yes') {
             $is_balancer = 1;
             $offloadto = &this_host_spares($dom_in_use);
         }
+        unless (defined($cached)) {
+            $domneedscache = $serverhomedom;
+        }
     }
-    my $lowest_load = 30000;
-    if (ref($offloadto) eq 'HASH') {
-        if (ref($offloadto->{'primary'}) eq 'ARRAY') {
-            foreach my $try_server (@{$offloadto->{'primary'}}) {
-                ($otherserver,$lowest_load) =
-                    &compare_server_load($try_server,$otherserver,$lowest_load);
+    if ($domneedscache) {
+        &do_cache_new('loadbalancing',$domneedscache,$is_balancer,$cachetime);
+    }
+    if ($is_balancer) {
+        my $lowest_load = 30000;
+        if (ref($offloadto) eq 'HASH') {
+            if (ref($offloadto->{'primary'}) eq 'ARRAY') {
+                foreach my $try_server (@{$offloadto->{'primary'}}) {
+                    ($otherserver,$lowest_load) =
+                        &compare_server_load($try_server,$otherserver,$lowest_load);
+                }
             }
-        }
-        my $found_server = ($otherserver ne '' && $lowest_load < 100);
+            my $found_server = ($otherserver ne '' && $lowest_load < 100);
 
-        if (!$found_server) {
-            if (ref($offloadto->{'default'}) eq 'ARRAY') {
-                foreach my $try_server (@{$offloadto->{'default'}}) {
+            if (!$found_server) {
+                if (ref($offloadto->{'default'}) eq 'ARRAY') {
+                    foreach my $try_server (@{$offloadto->{'default'}}) {
+                        ($otherserver,$lowest_load) =
+                            &compare_server_load($try_server,$otherserver,$lowest_load);
+                    }
+                }
+            }
+        } elsif (ref($offloadto) eq 'ARRAY') {
+            if (@{$offloadto} == 1) {
+                $otherserver = $offloadto->[0];
+            } elsif (@{$offloadto} > 1) {
+                foreach my $try_server (@{$offloadto}) {
                     ($otherserver,$lowest_load) =
                         &compare_server_load($try_server,$otherserver,$lowest_load);
                 }
             }
         }
-    } elsif (ref($offloadto) eq 'ARRAY') {
-        if (@{$offloadto} == 1) {
-            $otherserver = $offloadto->[0];
-        } elsif (@{$offloadto} > 1) {
-            foreach my $try_server (@{$offloadto}) {
-                ($otherserver,$lowest_load) =
-                    &compare_server_load($try_server,$otherserver,$lowest_load);
+        unless ($caller eq 'login') {
+            if (($otherserver ne '') && (grep(/^\Q$otherserver\E$/,@hosts))) {
+                $is_balancer = 0;
+                if ($uname ne '' && $udom ne '') {
+                    if (($env{'user.name'} eq $uname) && ($env{'user.domain'} eq $udom)) {
+                    
+                        &appenv({'user.loadbalexempt'     => $lonhost,  
+                                 'user.loadbalcheck.time' => time});
+                    }
+                }
             }
         }
     }
     return ($is_balancer,$otherserver);
 }
 
+sub check_balancer_result {
+    my ($result,@hosts) = @_;
+    my ($is_balancer,$currtargets,$currrules);
+    if (ref($result) eq 'HASH') {
+        if ($result->{'lonhost'} ne '') {
+            my $currbalancer = $result->{'lonhost'};
+            if (grep(/^\Q$currbalancer\E$/,@hosts)) {
+                $is_balancer = 1;
+                $currtargets = $result->{'targets'};
+                $currrules = $result->{'rules'};
+            }
+        } else {
+            foreach my $key (keys(%{$result})) {
+                if (($key ne '') && (grep(/^\Q$key\E$/,@hosts)) &&
+                    (ref($result->{$key}) eq 'HASH')) {
+                    $is_balancer = 1;
+                    $currrules = $result->{$key}{'rules'};
+                    $currtargets = $result->{$key}{'targets'};
+                    last;
+                }
+            }
+        }
+    }
+    return ($is_balancer,$currtargets,$currrules);
+}
+
 sub get_loadbalancer_targets {
     my ($rule_in_effect,$currtargets,$uname,$udom) = @_;
     my $offloadto;
-    if ($rule_in_effect eq '') {
+    if ($rule_in_effect eq 'none') {
+        return [$perlvar{'lonHostID'}];
+    } elsif ($rule_in_effect eq '') {
         $offloadto = $currtargets;
     } else {
         if ($rule_in_effect eq 'homeserver') {
@@ -1401,7 +1552,7 @@ sub get_loadbalancer_targets {
                     }
                 }
             } else {
-                my %servers = &dom_servers($udom);
+                my %servers = &internet_dom_servers($udom);
                 my ($remotebalancer,$remotetargets) = &get_lonbalancer_config(\%servers);
                 if (&hostname($remotebalancer) ne '') {
                     $offloadto = [$remotebalancer];
@@ -1447,6 +1598,146 @@ sub internet_dom_servers {
     return %uniqservers;
 }
 
+sub notcallable {
+    my ($cmdtype,$calldom) = @_;
+    if (&domain($calldom) eq '') {
+        return 1;
+    }
+    unless ($cmdtype =~ /^(content|shared|enroll|coaurem|domroles|catalog|reqcrs|msg)$/) {
+        return 1;
+    }
+    my @machinedoms = &current_machine_domains();
+    if (grep(/^\Q$calldom\E$/,@machinedoms)) {
+        return;
+    }
+    my $reject;
+    my $intdom = &internet_dom($perlvar{'lonHostID'});
+    if ($intdom eq '') {
+        return 1;
+    }
+    my $callprimary = &domain($calldom,'primary');
+    my $intcalldom = &Apache::lonnet::internet_dom($callprimary);
+    unless ($intdom eq $intcalldom) {
+        my ($trustconfig,$cached)=&Apache::lonnet::is_cached_new('trust',$calldom);
+        unless (defined($cached)) {
+            my %domconfig = &Apache::lonnet::get_dom('configuration',['trust'],$calldom);
+            &Apache::lonnet::do_cache_new('trust',$calldom,$domconfig{'trust'},3600);
+            $trustconfig = $domconfig{'trust'};
+        }
+        if (ref($trustconfig)) {
+            if (ref($trustconfig->{$cmdtype}) eq 'HASH') {
+                if (ref($trustconfig->{$cmdtype}->{'exc'}) eq 'ARRAY') {
+                    if (grep(/^\Q$intdom\E$/,@{$trustconfig->{$cmdtype}->{'exc'}})) {
+                        $reject = 1;
+                    }
+                }
+                if (ref($trustconfig->{$cmdtype}->{'inc'}) eq 'ARRAY') {
+                    if (grep(/^\Q$intdom\E$/,@{$trustconfig->{$cmdtype}->{'inc'}})) {
+                        $reject = 0;
+                    } else {
+                        $reject = 1;
+                    }
+                }
+            }
+        }
+    }
+    return $reject;
+}
+
+sub trusted_domains {
+    my ($cmdtype,$calldom) = @_;
+    my (%trusted,%untrusted);
+    if (&domain($calldom) eq '') {
+        return (\%trusted,\%untrusted);
+    }
+    unless ($cmdtype =~ /^(content|shared|enroll|coaurem|domroles|catalog|reqcrs|msg)$/) {
+        return (\%trusted,\%untrusted);
+    }
+    my $callprimary = &domain($calldom,'primary');
+    my $intcalldom = &Apache::lonnet::internet_dom($callprimary);
+    if ($intcalldom eq '') {
+        return (\%trusted,\%untrusted);
+    }
+
+    my ($trustconfig,$cached)=&Apache::lonnet::is_cached_new('trust',$calldom);
+    unless (defined($cached)) {
+        my %domconfig = &Apache::lonnet::get_dom('configuration',['trust'],$calldom);
+        &Apache::lonnet::do_cache_new('trust',$calldom,$domconfig{'trust'},3600);
+        $trustconfig = $domconfig{'trust'};
+    }
+    if (ref($trustconfig)) {
+        my (%possexc,%possinc,@allexc,@allinc); 
+        if (ref($trustconfig->{$cmdtype}) eq 'HASH') {
+            if (ref($trustconfig->{$cmdtype}->{'exc'}) eq 'ARRAY') {
+                map { $possexc{$_} = 1; } @{$trustconfig->{$cmdtype}->{'exc'}}; 
+            }
+            if (ref($trustconfig->{$cmdtype}->{'inc'}) eq 'ARRAY') {
+                map { $possinc{$_} = 1; } @{$trustconfig->{$cmdtype}->{'inc'}};
+            }
+        }
+        if (keys(%possexc)) {
+            if (keys(%possinc)) {
+                foreach my $key (sort(keys(%possexc))) {
+                    next if ($key eq $intcalldom);
+                    unless ($possinc{$key}) {
+                        push(@allexc,$key);
+                    }
+                }
+            } else {
+                @allexc = sort(keys(%possexc));
+            }
+        }
+        if (keys(%possinc)) {
+            $possinc{$intcalldom} = 1;
+            @allinc = sort(keys(%possinc));
+        }
+        if ((@allexc > 0) || (@allinc > 0)) {
+            my %doms_by_intdom;
+            my %allintdoms = &all_host_intdom();
+            my %alldoms = &all_host_domain();
+            foreach my $key (%allintdoms) {
+                if (ref($doms_by_intdom{$allintdoms{$key}}) eq 'ARRAY') {
+                    unless (grep(/^\Q$alldoms{$key}\E$/,@{$doms_by_intdom{$allintdoms{$key}}})) {
+                        push(@{$doms_by_intdom{$allintdoms{$key}}},$alldoms{$key});
+                    }
+                } else {
+                    $doms_by_intdom{$allintdoms{$key}} = [$alldoms{$key}]; 
+                }
+            }
+            foreach my $exc (@allexc) {
+                if (ref($doms_by_intdom{$exc}) eq 'ARRAY') {
+                    map { $untrusted{$_}; } @{$doms_by_intdom{$exc}};
+                }
+            }
+            foreach my $inc (@allinc) {
+                if (ref($doms_by_intdom{$inc}) eq 'ARRAY') {
+                    map { $trusted{$_}; } @{$doms_by_intdom{$inc}};
+                }
+            }
+        }
+    }
+    return(\%trusted,\%untrusted);
+}
+
+sub will_trust {
+    my ($cmdtype,$domain,$possdom) = @_;
+    return 1 if ($domain eq $possdom);
+    my ($trustedref,$untrustedref) = &trusted_domains($cmdtype,$possdom);
+    my $willtrust; 
+    if ((ref($trustedref) eq 'ARRAY') && (@{$trustedref} > 0)) {
+        if (grep(/^\Q$domain\E$/,@{$trustedref})) {
+            $willtrust = 1;
+        }
+    } elsif ((ref($untrustedref) eq 'ARRAY') && (@{$untrustedref} > 0)) {
+        unless (grep(/^\Q$domain\E$/,@{$untrustedref})) {
+            $willtrust = 1;
+        }
+    } else {
+        $willtrust = 1;
+    }
+    return $willtrust;
+}
+
 # ---------------------- Find the homebase for a user from domain's lib servers
 
 my %homecache;
@@ -1472,17 +1763,33 @@ sub homeserver {
     return 'no_host';
 }
 
-# ------------------------------------- Find the usernames behind a list of IDs
+# ----- Find the usernames behind a list of student/employee IDs or clicker IDs
 
 sub idget {
-    my ($udom,@ids)=@_;
+    my ($udom,$idsref,$namespace)=@_;
     my %returnhash=();
+    my @ids=(); 
+    if (ref($idsref) eq 'ARRAY') {
+        @ids = @{$idsref};
+    } else {
+        return %returnhash; 
+    }
+    if ($namespace eq '') {
+        $namespace = 'ids';
+    }
     
     my %servers = &get_servers($udom,'library');
     foreach my $tryserver (keys(%servers)) {
-	my $idlist=join('&',@ids);
-	$idlist=~tr/A-Z/a-z/; 
-	my $reply=&reply("idget:$udom:".$idlist,$tryserver);
+	my $idlist=join('&', map { &escape($_); } @ids);
+	if ($namespace eq 'ids') {
+	    $idlist=~tr/A-Z/a-z/;
+	}
+	my $reply;
+	if ($namespace eq 'ids') {
+	    $reply=&reply("idget:$udom:".$idlist,$tryserver);
+	} else {
+	    $reply=&reply("getdom:$udom:$namespace:$idlist",$tryserver);
+	}
 	my @answer=();
 	if (($reply ne 'con_lost') && ($reply!~/^error\:/)) {
 	    @answer=split(/\&/,$reply);
@@ -1490,10 +1797,10 @@ sub idget {
 	my $i;
 	for ($i=0;$i<=$#ids;$i++) {
 	    if ($answer[$i]) {
-		$returnhash{$ids[$i]}=$answer[$i];
-	    } 
+		$returnhash{$ids[$i]}=&unescape($answer[$i]);
+	    }
 	}
-    } 
+    }
     return %returnhash;
 }
 
@@ -1508,48 +1815,157 @@ sub idrget {
     return %returnhash;
 }
 
-# ------------------------------- Store away a list of names and associated IDs
+# Store away a list of names and associated student/employee IDs or clicker IDs
 
 sub idput {
-    my ($udom,%ids)=@_;
+    my ($udom,$idsref,$uhom,$namespace)=@_;
     my %servers=();
+    my %ids=();
+    my %byid = ();
+    if (ref($idsref) eq 'HASH') {
+        %ids=%{$idsref};
+    }
+    if ($namespace eq '') {
+        $namespace = 'ids'; 
+    }
     foreach my $uname (keys(%ids)) {
 	&cput('environment',{'id'=>$ids{$uname}},$udom,$uname);
-        my $uhom=&homeserver($uname,$udom);
+        if ($uhom eq '') {
+            $uhom=&homeserver($uname,$udom);
+        }
         if ($uhom ne 'no_host') {
-            my $id=&escape($ids{$uname});
-            $id=~tr/A-Z/a-z/;
             my $esc_unam=&escape($uname);
-	    if ($servers{$uhom}) {
-		$servers{$uhom}.='&'.$id.'='.$esc_unam;
+            if ($namespace eq 'ids') {
+                my $id=&escape($ids{$uname});
+                $id=~tr/A-Z/a-z/;
+                my $esc_unam=&escape($uname);
+                $servers{$uhom}.=$id.'='.$esc_unam.'&';
             } else {
-                $servers{$uhom}=$id.'='.$esc_unam;
+                my @currids = split(/,/,$ids{$uname});
+                foreach my $id (@currids) {
+                    $byid{$uhom}{$id} .= $uname.',';
+                }
+            }
+        }
+    }
+    if ($namespace eq 'clickers') {
+        foreach my $server (keys(%byid)) {
+            if (ref($byid{$server}) eq 'HASH') {
+                foreach my $id (keys(%{$byid{$server}})) {
+                    $byid{$server} =~ s/,$//;
+                    $servers{$uhom}.=&escape($id).'='.&escape($byid{$server}).'&'; 
+                }
             }
         }
     }
     foreach my $server (keys(%servers)) {
-        &critical('idput:'.$udom.':'.$servers{$server},$server);
+        $servers{$server} =~ s/\&$//;
+        if ($namespace eq 'ids') {     
+            &critical('idput:'.$udom.':'.$servers{$server},$server);
+        } else {
+            &critical('updateclickers:'.$udom.':add:'.$servers{$server},$server);
+        }
     }
 }
 
-# ------------------------------dump from db file owned by domainconfig user
-sub dump_dom {
-    my ($namespace,$udom,$regexp,$range)=@_;
-    if (!$udom) {
-        $udom=$env{'user.domain'};
+# ------------- Delete unwanted student/employee IDs or clicker IDs from domain
+
+sub iddel {
+    my ($udom,$idshashref,$uhome,$namespace)=@_;
+    my %result=();
+    my %ids=();
+    my %byid = ();
+    if (ref($idshashref) eq 'HASH') {
+        %ids=%{$idshashref};
+    } else {
+        return %result;
     }
-    my %returnhash;
-    if ($udom) {
-        my $uname = &get_domainconfiguser($udom);
-        %returnhash = &dump($namespace,$udom,$uname,$regexp,$range);
+    if ($namespace eq '') {
+        $namespace = 'ids';
+    }
+    my %servers=();
+    while (my ($id,$unamestr) = each(%ids)) {
+        if ($namespace eq 'ids') {
+            my $uhom = $uhome;
+            if ($uhom eq '') { 
+                $uhom=&homeserver($unamestr,$udom);
+            }
+            if ($uhom ne 'no_host') {
+                $servers{$uhom}.='&'.&escape($id);
+            }
+         } else {
+            my @curritems = split(/,/,$ids{$id});
+            foreach my $uname (@curritems) {
+                my $uhom = $uhome;
+                if ($uhom eq '') {
+                    $uhom=&homeserver($uname,$udom);
+                }
+                if ($uhom ne 'no_host') { 
+                    $byid{$uhom}{$id} .= $uname.',';
+                }
+            }
+        }
+    }
+    if ($namespace eq 'clickers') {
+        foreach my $server (keys(%byid)) {
+            if (ref($byid{$server}) eq 'HASH') {
+                foreach my $id (keys(%{$byid{$server}})) {
+                    $byid{$server}{$id} =~ s/,$//;
+                    $servers{$server}.=&escape($id).'='.&escape($byid{$server}{$id}).'&';
+                }
+            }
+        }
+    }
+    foreach my $server (keys(%servers)) {
+        $servers{$server} =~ s/\&$//;
+        if ($namespace eq 'ids') {
+            $result{$server} = &critical('iddel:'.$udom.':'.$servers{$server},$uhome);
+        } elsif ($namespace eq 'clickers') {
+            $result{$server} = &critical('updateclickers:'.$udom.':del:'.$servers{$server},$server);
+        }
+    }
+    return %result;
+}
+
+# ----- Update clicker ID-to-username look-ups in clickers.db on library server 
+
+sub updateclickers {
+    my ($udom,$action,$idshashref,$uhome,$critical) = @_;
+    my %clickers;
+    if (ref($idshashref) eq 'HASH') {
+        %clickers=%{$idshashref};
+    } else {
+        return;
+    }
+    my $items='';
+    foreach my $item (keys(%clickers)) {
+        $items.=&escape($item).'='.&escape($clickers{$item}).'&';
+    }
+    $items=~s/\&$//;
+    my $request = "updateclickers:$udom:$action:$items";
+    if ($critical) {
+        return &critical($request,$uhome);
+    } else {
+        return &reply($request,$uhome);
     }
-    return %returnhash;
+}
+
+# ------------------------------dump from db file owned by domainconfig user
+sub dump_dom {
+    my ($namespace, $udom, $regexp) = @_;
+
+    $udom ||= $env{'user.domain'};
+
+    return () unless $udom;
+
+    return &dump($namespace, $udom, &get_domainconfiguser($udom), $regexp);
 }
 
 # ------------------------------------------ get items from domain db files   
 
 sub get_dom {
     my ($namespace,$storearr,$udom,$uhome)=@_;
+    return if ($udom eq 'public');
     my $items='';
     foreach my $item (@$storearr) {
         $items.=&escape($item).'&';
@@ -1557,6 +1973,7 @@ sub get_dom {
     $items=~s/\&$//;
     if (!$udom) {
         $udom=$env{'user.domain'};
+        return if ($udom eq 'public');
         if (defined(&domain($udom,'primary'))) {
             $uhome=&domain($udom,'primary');
         } else {
@@ -1570,7 +1987,12 @@ sub get_dom {
         }
     }
     if ($udom && $uhome && ($uhome ne 'no_host')) {
-        my $rep=&reply("getdom:$udom:$namespace:$items",$uhome);
+        my $rep;
+        if ($namespace =~ /^enc/) {
+            $rep=&reply("encrypt:egetdom:$udom:$namespace:$items",$uhome);
+        } else {
+            $rep=&reply("getdom:$udom:$namespace:$items",$uhome);
+        }
         my %returnhash;
         if ($rep eq '' || $rep =~ /^error: 2 /) {
             return %returnhash;
@@ -1614,7 +2036,11 @@ sub put_dom {
             $items.=&escape($item).'='.&freeze_escape($$storehash{$item}).'&';
         }
         $items=~s/\&$//;
-        return &reply("putdom:$udom:$namespace:$items",$uhome);
+        if ($namespace =~ /^enc/) {
+            return &reply("encrypt:putdom:$udom:$namespace:$items",$uhome);
+        } else {
+            return &reply("putdom:$udom:$namespace:$items",$uhome);
+        }
     } else {
         &logthis("put_dom failed - no homeserver and/or domain");
     }
@@ -1660,14 +2086,13 @@ sub retrieve_inst_usertypes {
     my %domdefs = &Apache::lonnet::get_domain_defaults($udom);
     if ((ref($domdefs{'inststatustypes'}) eq 'HASH') && 
         (ref($domdefs{'inststatusorder'}) eq 'ARRAY')) {
-        %returnhash = %{$domdefs{'inststatustypes'}};
-        @order = @{$domdefs{'inststatusorder'}};
+        return ($domdefs{'inststatustypes'},$domdefs{'inststatusorder'});
     } else {
         if (defined(&domain($udom,'primary'))) {
             my $uhome=&domain($udom,'primary');
             my $rep=&reply("inst_usertypes:$udom",$uhome);
             if ($rep =~ /^(con_lost|error|no_such_host|refused)/) {
-                &logthis("get_dom failed - $rep returned from $uhome in domain: $udom");
+                &logthis("retrieve_inst_usertypes failed - $rep returned from $uhome in domain: $udom");
                 return (\%returnhash,\@order);
             }
             my ($hashitems,$orderitems) = split(/:/,$rep); 
@@ -1683,15 +2108,15 @@ sub retrieve_inst_usertypes {
                 push(@order,&unescape($item));
             }
         } else {
-            &logthis("get_dom failed - no primary domain server for $udom");
+            &logthis("retrieve_inst_usertypes failed - no primary domain server for $udom");
         }
+        return (\%returnhash,\@order);
     }
-    return (\%returnhash,\@order);
 }
 
 sub is_domainimage {
     my ($url) = @_;
-    if ($url=~m-^/+res/+($match_domain)/+\1\-domainconfig/+(img|logo|domlogo)/+-) {
+    if ($url=~m-^/+res/+($match_domain)/+\1\-domainconfig/+(img|logo|domlogo)/+[^/]-) {
         if (&domain($1) ne '') {
             return '1';
         }
@@ -1712,7 +2137,7 @@ sub inst_directory_query {
 			   &escape($srch->{'srchtype'}),$homeserver);
 	my $host=&hostname($homeserver);
 	if ($queryid !~/^\Q$host\E\_/) {
-	    &logthis('instituional directory search invalid queryid: '.$queryid.' for host: '.$homeserver.'in domain '.$udom);
+	    &logthis('institutional directory search invalid queryid: '.$queryid.' for host: '.$homeserver.' in domain '.$udom);
 	    return;
 	}
 	my $response = &get_query_reply($queryid);
@@ -1832,6 +2257,63 @@ sub get_instuser {
     return ($outcome,%userinfo);
 }
 
+sub get_multiple_instusers {
+    my ($udom,$users,$caller) = @_;
+    my ($outcome,$results);
+    if (ref($users) eq 'HASH') {
+        my $count = keys(%{$users}); 
+        my $requested = &freeze_escape($users);
+        my $homeserver = &domain($udom,'primary');
+        if ($homeserver ne '') {
+            my $queryid=&reply('querysend:getmultinstusers:::'.$caller.'='.$requested,$homeserver);
+            my $host=&hostname($homeserver);
+            if ($queryid !~/^\Q$host\E\_/) {
+                &logthis('get_multiple_instusers invalid queryid: '.$queryid.
+                         ' for host: '.$homeserver.'in domain '.$udom);
+                return ($outcome,$results);
+            }
+            my $response = &get_query_reply($queryid);
+            my $maxtries = 5;
+            if ($count > 100) {
+                $maxtries = 1+int($count/20);
+            }
+            my $tries = 1;
+            while (($response=~/^timeout/) && ($tries <= $maxtries)) {
+                $response = &get_query_reply($queryid);
+                $tries ++;
+            }
+            if ($response eq '') {
+                $results = {};
+                foreach my $key (keys(%{$users})) {
+                    my ($uname,$id);
+                    if ($caller eq 'id') {
+                        $id = $key;
+                    } else {
+                        $uname = $key;
+                    }
+                    my ($resp,%info) = &get_instuser($udom,$uname,$id);
+                    $outcome = $resp;
+                    if ($resp eq 'ok') {
+                        %{$results} = (%{$results}, %info);
+                    } else {
+                        last;
+                    }
+                }
+            } elsif(!&error($response) && ($response ne 'refused')) {
+                if (($response eq 'unavailable') || ($response eq 'invalid') || ($response eq 'timeout')) {
+                    $outcome = $response;
+                } else {
+                    ($outcome,my $userdata) = split(/=/,$response,2);
+                    if ($outcome eq 'ok') {
+                        $results = &thaw_unescape($userdata); 
+                    }
+                }
+            }
+        }
+    }
+    return ($outcome,$results);
+}
+
 sub inst_rulecheck {
     my ($udom,$uname,$id,$item,$rules) = @_;
     my %returnhash;
@@ -1911,19 +2393,26 @@ sub inst_userrules {
 # ------------- Get Authentication, Language and User Tools Defaults for Domain
 
 sub get_domain_defaults {
-    my ($domain) = @_;
+    my ($domain,$ignore_cache) = @_;
+    return if (($domain eq '') || ($domain eq 'public'));
     my $cachetime = 60*60*24;
-    my ($result,$cached)=&is_cached_new('domdefaults',$domain);
-    if (defined($cached)) {
-        if (ref($result) eq 'HASH') {
-            return %{$result};
+    unless ($ignore_cache) {
+        my ($result,$cached)=&is_cached_new('domdefaults',$domain);
+        if (defined($cached)) {
+            if (ref($result) eq 'HASH') {
+                return %{$result};
+            }
         }
     }
     my %domdefaults;
     my %domconfig =
          &Apache::lonnet::get_dom('configuration',['defaults','quotas',
                                   'requestcourses','inststatus',
-                                  'coursedefaults','usersessions'],$domain);
+                                  'coursedefaults','usersessions',
+                                  'requestauthor','selfenrollment',
+                                  'coursecategories','ssl','autoenroll',
+                                  'trust','helpsettings'],$domain);
+    my @coursetypes = ('official','unofficial','community','textbook','placement');
     if (ref($domconfig{'defaults'}) eq 'HASH') {
         $domdefaults{'lang_def'} = $domconfig{'defaults'}{'lang_def'}; 
         $domdefaults{'auth_def'} = $domconfig{'defaults'}{'auth_def'};
@@ -1931,6 +2420,9 @@ sub get_domain_defaults {
         $domdefaults{'timezone_def'} = $domconfig{'defaults'}{'timezone_def'};
         $domdefaults{'datelocale_def'} = $domconfig{'defaults'}{'datelocale_def'};
         $domdefaults{'portal_def'} = $domconfig{'defaults'}{'portal_def'};
+        $domdefaults{'intauth_cost'} = $domconfig{'defaults'}{'intauth_cost'};
+        $domdefaults{'intauth_switch'} = $domconfig{'defaults'}{'intauth_switch'};
+        $domdefaults{'intauth_check'} = $domconfig{'defaults'}{'intauth_check'};
     } else {
         $domdefaults{'lang_def'} = &domain($domain,'lang_def');
         $domdefaults{'auth_def'} = &domain($domain,'auth_def');
@@ -1941,27 +2433,62 @@ sub get_domain_defaults {
             $domdefaults{'defaultquota'} = $domconfig{'quotas'}{'defaultquota'};
         } else {
             $domdefaults{'defaultquota'} = $domconfig{'quotas'};
-        } 
-        my @usertools = ('aboutme','blog','portfolio');
+        }
+        my @usertools = ('aboutme','blog','webdav','portfolio');
         foreach my $item (@usertools) {
             if (ref($domconfig{'quotas'}{$item}) eq 'HASH') {
                 $domdefaults{$item} = $domconfig{'quotas'}{$item};
             }
         }
+        if (ref($domconfig{'quotas'}{'authorquota'}) eq 'HASH') {
+            $domdefaults{'authorquota'} = $domconfig{'quotas'}{'authorquota'};
+        }
     }
     if (ref($domconfig{'requestcourses'}) eq 'HASH') {
-        foreach my $item ('official','unofficial','community') {
+        foreach my $item ('official','unofficial','community','textbook','placement') {
             $domdefaults{$item} = $domconfig{'requestcourses'}{$item};
         }
     }
+    if (ref($domconfig{'requestauthor'}) eq 'HASH') {
+        $domdefaults{'requestauthor'} = $domconfig{'requestauthor'};
+    }
     if (ref($domconfig{'inststatus'}) eq 'HASH') {
-        foreach my $item ('inststatustypes','inststatusorder') {
+        foreach my $item ('inststatustypes','inststatusorder','inststatusguest') {
             $domdefaults{$item} = $domconfig{'inststatus'}{$item};
         }
     }
     if (ref($domconfig{'coursedefaults'}) eq 'HASH') {
-        foreach my $item ('canuse_pdfforms') {
-            $domdefaults{$item} = $domconfig{'coursedefaults'}{$item};
+        $domdefaults{'canuse_pdfforms'} = $domconfig{'coursedefaults'}{'canuse_pdfforms'};
+        $domdefaults{'usejsme'} = $domconfig{'coursedefaults'}{'usejsme'};
+        $domdefaults{'uselcmath'} = $domconfig{'coursedefaults'}{'uselcmath'};
+        if (ref($domconfig{'coursedefaults'}{'postsubmit'}) eq 'HASH') {
+            $domdefaults{'postsubmit'} = $domconfig{'coursedefaults'}{'postsubmit'}{'client'};
+        }
+        foreach my $type (@coursetypes) {
+            if (ref($domconfig{'coursedefaults'}{'coursecredits'}) eq 'HASH') {
+                unless ($type eq 'community') {
+                    $domdefaults{$type.'credits'} = $domconfig{'coursedefaults'}{'coursecredits'}{$type};
+                }
+            }
+            if (ref($domconfig{'coursedefaults'}{'uploadquota'}) eq 'HASH') {
+                $domdefaults{$type.'quota'} = $domconfig{'coursedefaults'}{'uploadquota'}{$type};
+            }
+            if ($domdefaults{'postsubmit'} eq 'on') {
+                if (ref($domconfig{'coursedefaults'}{'postsubmit'}{'timeout'}) eq 'HASH') {
+                    $domdefaults{$type.'postsubtimeout'} = 
+                        $domconfig{'coursedefaults'}{'postsubmit'}{'timeout'}{$type}; 
+                }
+            }
+        }
+        if (ref($domconfig{'coursedefaults'}{'canclone'}) eq 'HASH') {
+            if (ref($domconfig{'coursedefaults'}{'canclone'}{'instcode'}) eq 'ARRAY') {
+                my @clonecodes = @{$domconfig{'coursedefaults'}{'canclone'}{'instcode'}};
+                if (@clonecodes) {
+                    $domdefaults{'canclone'} = join('+',@clonecodes);
+                }
+            }
+        } elsif ($domconfig{'coursedefaults'}{'canclone'}) {
+            $domdefaults{'canclone'}=$domconfig{'coursedefaults'}{'canclone'};
         }
     }
     if (ref($domconfig{'usersessions'}) eq 'HASH') {
@@ -1971,12 +2498,96 @@ sub get_domain_defaults {
         if (ref($domconfig{'usersessions'}{'hosted'}) eq 'HASH') {
             $domdefaults{'hostedsessions'} = $domconfig{'usersessions'}{'hosted'};
         }
+        if (ref($domconfig{'usersessions'}{'offloadnow'}) eq 'HASH') {
+            $domdefaults{'offloadnow'} = $domconfig{'usersessions'}{'offloadnow'};
+        }
     }
-    &Apache::lonnet::do_cache_new('domdefaults',$domain,\%domdefaults,
-                                  $cachetime);
+    if (ref($domconfig{'selfenrollment'}) eq 'HASH') {
+        if (ref($domconfig{'selfenrollment'}{'admin'}) eq 'HASH') {
+            my @settings = ('types','registered','enroll_dates','access_dates','section',
+                            'approval','limit');
+            foreach my $type (@coursetypes) {
+                if (ref($domconfig{'selfenrollment'}{'admin'}{$type}) eq 'HASH') {
+                    my @mgrdc = ();
+                    foreach my $item (@settings) {
+                        if ($domconfig{'selfenrollment'}{'admin'}{$type}{$item} eq '0') {
+                            push(@mgrdc,$item);
+                        }
+                    }
+                    if (@mgrdc) {
+                        $domdefaults{$type.'selfenrolladmdc'} = join(',',@mgrdc);
+                    }
+                }
+            }
+        }
+        if (ref($domconfig{'selfenrollment'}{'default'}) eq 'HASH') {
+            foreach my $type (@coursetypes) {
+                if (ref($domconfig{'selfenrollment'}{'default'}{$type}) eq 'HASH') {
+                    foreach my $item (keys(%{$domconfig{'selfenrollment'}{'default'}{$type}})) {
+                        $domdefaults{$type.'selfenroll'.$item} = $domconfig{'selfenrollment'}{'default'}{$type}{$item};
+                    }
+                }
+            }
+        }
+    }
+    if (ref($domconfig{'coursecategories'}) eq 'HASH') {
+        $domdefaults{'catauth'} = 'std';
+        $domdefaults{'catunauth'} = 'std';
+        if ($domconfig{'coursecategories'}{'auth'}) { 
+            $domdefaults{'catauth'} = $domconfig{'coursecategories'}{'auth'};
+        }
+        if ($domconfig{'coursecategories'}{'unauth'}) {
+            $domdefaults{'catunauth'} = $domconfig{'coursecategories'}{'unauth'};
+        }
+    }
+    if (ref($domconfig{'ssl'}) eq 'HASH') {
+        if (ref($domconfig{'ssl'}{'replication'}) eq 'HASH') {
+            $domdefaults{'replication'} = $domconfig{'ssl'}{'replication'};
+        }
+        if (ref($domconfig{'ssl'}{'connto'}) eq 'HASH') {
+            $domdefaults{'connect'} = $domconfig{'ssl'}{'connto'};
+        }
+        if (ref($domconfig{'ssl'}{'connfrom'}) eq 'HASH') {
+            $domdefaults{'connect'} = $domconfig{'ssl'}{'connfrom'};
+        }
+    }
+    if (ref($domconfig{'trust'}) eq 'HASH') {
+        my @prefixes = qw(content shared enroll othcoau coaurem domroles catalog reqcrs msg);
+        foreach my $prefix (@prefixes) {
+            if (ref($domconfig{'trust'}{$prefix}) eq 'HASH') {
+                $domdefaults{'trust'.$prefix} = $domconfig{'trust'}{$prefix};
+            }
+        }
+    }
+    if (ref($domconfig{'autoenroll'}) eq 'HASH') {
+        $domdefaults{'autofailsafe'} = $domconfig{'autoenroll'}{'autofailsafe'};
+    }
+    if (ref($domconfig{'helpsettings'}) eq 'HASH') {
+        $domdefaults{'submitbugs'} = $domconfig{'helpsettings'}{'submitbugs'};
+        if (ref($domconfig{'helpsettings'}{'adhoc'}) eq 'HASH') {
+            $domdefaults{'adhocroles'} = $domconfig{'helpsettings'}{'adhoc'};
+        }
+    }
+    &do_cache_new('domdefaults',$domain,\%domdefaults,$cachetime);
     return %domdefaults;
 }
 
+sub course_portal_url {
+    my ($cnum,$cdom) = @_;
+    my $chome = &homeserver($cnum,$cdom);
+    my $hostname = &hostname($chome);
+    my $protocol = $protocol{$chome};
+    $protocol = 'http' if ($protocol ne 'https');
+    my %domdefaults = &get_domain_defaults($cdom);
+    my $firsturl;
+    if ($domdefaults{'portal_def'}) {
+        $firsturl = $domdefaults{'portal_def'};
+    } else {
+        $firsturl = $protocol.'://'.$hostname;
+    }
+    return $firsturl;
+}
+
 # --------------------------------------------------- Assign a key to a student
 
 sub assign_access_key {
@@ -2157,8 +2768,7 @@ sub getsection {
     # If there is a role which has expired, return it.
     #
     $courseid = &courseid_to_courseurl($courseid);
-    my $extra = &freeze_escape({'skipcheck' => 1});
-    my %roleshash = &dump('roles',$udom,$unam,$courseid,undef,$extra);
+    my %roleshash = &dump('roles',$udom,$unam,$courseid);
     foreach my $key (keys(%roleshash)) {
         next if ($key !~/^\Q$courseid\E(?:\/)*(\w+)*\_st$/);
         my $section=$1;
@@ -2214,21 +2824,23 @@ sub make_key {
 sub devalidate_cache_new {
     my ($name,$id,$debug) = @_;
     if ($debug) { &Apache::lonnet::logthis("deleting $name:$id"); }
+    my $remembered_id=$name.':'.$id;
     $id=&make_key($name,$id);
     $memcache->delete($id);
-    delete($remembered{$id});
-    delete($accessed{$id});
+    delete($remembered{$remembered_id});
+    delete($accessed{$remembered_id});
 }
 
 sub is_cached_new {
     my ($name,$id,$debug) = @_;
-    $id=&make_key($name,$id);
-    if (exists($remembered{$id})) {
-	if ($debug) { &Apache::lonnet::logthis("Early return $id of $remembered{$id} "); }
-	$accessed{$id}=[&gettimeofday()];
+    my $remembered_id=$name.':'.$id; # this is to avoid make_key (which is slow) whenever possible
+    if (exists($remembered{$remembered_id})) {
+	if ($debug) { &Apache::lonnet::logthis("Early return $remembered_id of $remembered{$remembered_id} "); }
+	$accessed{$remembered_id}=[&gettimeofday()];
 	$hits++;
-	return ($remembered{$id},1);
+	return ($remembered{$remembered_id},1);
     }
+    $id=&make_key($name,$id);
     my $value = $memcache->get($id);
     if (!(defined($value))) {
 	if ($debug) { &Apache::lonnet::logthis("getting $id is not defined"); }
@@ -2238,13 +2850,14 @@ sub is_cached_new {
 	if ($debug) { &Apache::lonnet::logthis("getting $id is __undef__"); }
 	$value=undef;
     }
-    &make_room($id,$value,$debug);
+    &make_room($remembered_id,$value,$debug);
     if ($debug) { &Apache::lonnet::logthis("getting $id is $value"); }
     return ($value,1);
 }
 
 sub do_cache_new {
     my ($name,$id,$value,$time,$debug) = @_;
+    my $remembered_id=$name.':'.$id;
     $id=&make_key($name,$id);
     my $setvalue=$value;
     if (!defined($setvalue)) {
@@ -2260,17 +2873,17 @@ sub do_cache_new {
 	$memcache->disconnect_all();
     }
     # need to make a copy of $value
-    &make_room($id,$value,$debug);
+    &make_room($remembered_id,$value,$debug);
     return $value;
 }
 
 sub make_room {
-    my ($id,$value,$debug)=@_;
+    my ($remembered_id,$value,$debug)=@_;
 
-    $remembered{$id}= (ref($value)) ? &Storable::dclone($value)
+    $remembered{$remembered_id}= (ref($value)) ? &Storable::dclone($value)
                                     : $value;
     if ($to_remember<0) { return; }
-    $accessed{$id}=[&gettimeofday()];
+    $accessed{$remembered_id}=[&gettimeofday()];
     if (scalar(keys(%remembered)) <= $to_remember) { return; }
     my $to_kick;
     my $max_time=0;
@@ -2384,7 +2997,7 @@ sub chatsend {
 
 sub getversion {
     my $fname=&clutter(shift);
-    unless ($fname=~/^\/res\//) { return -1; }
+    unless ($fname=~m{^(/adm/wrapper|)/res/}) { return -1; }
     return &currentversion(&filelocation('',$fname));
 }
 
@@ -2442,7 +3055,7 @@ sub repcopy {
     $filename=~s/\/+/\//g;
     my $londocroot = $perlvar{'lonDocRoot'};
     if ($filename=~m{^\Q$londocroot/adm/\E}) { return 'ok'; }
-    if ($filename=~m{^\Q$londocroot/lonUsers/\E}) { return 'ok'; }
+    if ($filename=~m{^\Q/home/httpd/lonUsers/\E}) { return 'ok'; }
     if ($filename=~m{^\Q$londocroot/userfiles/\E} or
 	$filename=~m{^/*(uploaded|editupload)/}) {
 	return &repcopy_userfile($filename);
@@ -2482,9 +3095,13 @@ sub repcopy {
 		   mkdir($path,0777);
                }
            }
-           my $ua=new LWP::UserAgent;
            my $request=new HTTP::Request('GET',"$remoteurl");
-           my $response=$ua->request($request,$transname);
+           my $response;
+           if ($remoteurl =~ m{/raw/}) {
+               $response=&LONCAPA::LWPReq::makerequest($home,$request,$transname,\%perlvar,'',0,1);
+           } else {
+               $response=&LONCAPA::LWPReq::makerequest($home,$request,$transname,\%perlvar,'',1);
+           }
            if ($response->is_error()) {
 	       unlink($transname);
                my $message=$response->status_line;
@@ -2494,7 +3111,12 @@ sub repcopy {
            } else {
 	       if ($remoteurl!~/\.meta$/) {
                   my $mrequest=new HTTP::Request('GET',$remoteurl.'.meta');
-                  my $mresponse=$ua->request($mrequest,$filename.'.meta');
+                  my $mresponse;
+                  if ($remoteurl =~ m{/raw/}) {
+                      $mresponse = &LONCAPA::LWPReq::makerequest($home,$mrequest,$filename.'.meta',\%perlvar,'',0,1);
+                  } else {
+                      $mresponse = &LONCAPA::LWPReq::makerequest($home,$mrequest,$filename.'.meta',\%perlvar,'',1);
+                  }
                   if ($mresponse->is_error()) {
 		      unlink($filename.'.meta');
                       &logthis(
@@ -2557,20 +3179,25 @@ sub absolute_url {
 sub ssi {
 
     my ($fn,%form)=@_;
-    my $ua=new LWP::UserAgent;
     my $request;
 
     $form{'no_update_last_known'}=1;
     &Apache::lonenc::check_encrypt(\$fn);
     if (%form) {
       $request=new HTTP::Request('POST',&absolute_url().$fn);
-      $request->content(join('&',map { &escape($_).'='.&escape($form{$_}) } keys(%form)));
+      $request->content(join('&',map { 
+            my $name = escape($_);
+            "$name=" . ( ref($form{$_}) eq 'ARRAY' 
+            ? join("&$name=", map {escape($_) } @{$form{$_}}) 
+            : &escape($form{$_}) );    
+        } keys(%form)));
     } else {
       $request=new HTTP::Request('GET',&absolute_url().$fn);
     }
 
     $request->header(Cookie => $ENV{'HTTP_COOKIE'});
-    my $response=$ua->request($request);
+    my $lonhost = $perlvar{'lonHostID'};
+    my $response= &LONCAPA::LWPReq::makerequest($lonhost,$request,'',\%perlvar);
 
     if (wantarray) {
 	return ($response->content, $response);
@@ -2581,9 +3208,8 @@ sub ssi {
 
 sub externalssi {
     my ($url)=@_;
-    my $ua=new LWP::UserAgent;
     my $request=new HTTP::Request('GET',$url);
-    my $response=$ua->request($request);
+    my $response = &LONCAPA::LWPReq::makerequest('',$request,'',\%perlvar);
     if (wantarray) {
         return ($response->content, $response);
     } else {
@@ -2604,6 +3230,285 @@ sub allowuploaded {
     &Apache::lonnet::appenv(\%httpref);
 }
 
+#
+# Determine if the current user should be able to edit a particular resource,
+# when viewing in course context.
+# (a) When viewing resource used to determine if "Edit" item is included in 
+#     Functions.
+# (b) When displaying folder contents in course editor, used to determine if
+#     "Edit" link will be displayed alongside resource.
+#
+#  input: six args -- filename (decluttered), course number, course domain,
+#                   url, symb (if registered) and group (if this is a group
+#                   item -- e.g., bulletin board, group page etc.).
+#  output: array of five scalars -- 
+#          $cfile -- url for file editing if editable on current server
+#          $home -- homeserver of resource (i.e., for author if published,
+#                                           or course if uploaded.).
+#          $switchserver --  1 if server switch will be needed.
+#          $forceedit -- 1 if icon/link should be to go to edit mode 
+#          $forceview -- 1 if icon/link should be to go to view mode
+#
+
+sub can_edit_resource {
+    my ($file,$cnum,$cdom,$resurl,$symb,$group) = @_;
+    my ($cfile,$home,$switchserver,$forceedit,$forceview,$uploaded,$incourse);
+#
+# For aboutme pages user can only edit his/her own.
+#
+    if ($resurl =~ m{^/?adm/($match_domain)/($match_username)/aboutme$}) {
+        my ($sdom,$sname) = ($1,$2);
+        if (($sdom eq $env{'user.domain'}) && ($sname eq $env{'user.name'})) {
+            $home = $env{'user.home'};
+            $cfile = $resurl;
+            if ($env{'form.forceedit'}) {
+                $forceview = 1;
+            } else {
+                $forceedit = 1;
+            }
+            return ($cfile,$home,$switchserver,$forceedit,$forceview);
+        } else {
+            return;
+        }
+    }
+
+    if ($env{'request.course.id'}) {
+        my $crsedit = &Apache::lonnet::allowed('mdc',$env{'request.course.id'});
+        if ($group ne '') {
+# if this is a group homepage or group bulletin board, check group privs
+            my $allowed = 0;
+            if ($resurl =~ m{^/?adm/$cdom/$cnum/$group/smppg$}) {
+                if ((&allowed('mdg',$env{'request.course.id'}.
+                              ($env{'request.course.sec'}?'/'.$env{'request.course.sec'}:''))) ||
+                        (&allowed('mgh',$env{'request.course.id'}.'/'.$group)) || $crsedit) {
+                    $allowed = 1;
+                }
+            } elsif ($resurl =~ m{^/?adm/$cdom/$cnum/\d+/bulletinboard$}) {
+                if ((&allowed('mdg',$env{'request.course.id'}.($env{'request.course.sec'}?'/'.$env{'request.course.sec'}:''))) ||
+                        (&allowed('cgb',$env{'request.course.id'}.'/'.$group)) || $crsedit) {
+                    $allowed = 1;
+                }
+            }
+            if ($allowed) {
+                $home=&homeserver($cnum,$cdom);
+                if ($env{'form.forceedit'}) {
+                    $forceview = 1;
+                } else {
+                    $forceedit = 1;
+                }
+                $cfile = $resurl;
+            } else {
+                return;
+            }
+        } else {
+            if ($resurl =~ m{^/?adm/viewclasslist$}) {
+                unless (&Apache::lonnet::allowed('opa',$env{'request.course.id'})) {
+                    return;
+                }
+            } elsif (!$crsedit) {
+#
+# No edit allowed where CC has switched to student role.
+#
+                return;
+            }
+        }
+    }
+
+    if ($file ne '') {
+        if (($cnum =~ /$match_courseid/) && ($cdom =~ /$match_domain/)) {
+            if (&is_course_upload($file,$cnum,$cdom)) {
+                $uploaded = 1;
+                $incourse = 1;
+                if ($file =~/\.(htm|html|css|js|txt)$/) {
+                    $cfile = &hreflocation('',$file);
+                    if ($env{'form.forceedit'}) {
+                        $forceview = 1;
+                    } else {
+                        $forceedit = 1;
+                    }
+                }
+            } elsif ($resurl =~ m{^/public/$cdom/$cnum/syllabus}) {
+                $incourse = 1;
+                if ($env{'form.forceedit'}) {
+                    $forceview = 1;
+                } else {
+                    $forceedit = 1;
+                }
+                $cfile = $resurl;
+            } elsif (($resurl ne '') && (&is_on_map($resurl))) { 
+                if ($resurl =~ m{^/adm/$match_domain/$match_username/\d+/smppg|bulletinboard$}) {
+                    $incourse = 1;
+                    if ($env{'form.forceedit'}) {
+                        $forceview = 1;
+                    } else {
+                        $forceedit = 1;
+                    }
+                    $cfile = $resurl;
+                } elsif ($resurl eq '/res/lib/templates/simpleproblem.problem') {
+                    $incourse = 1;
+                    $cfile = $resurl.'/smpedit';
+                } elsif ($resurl =~ m{^/adm/wrapper/ext/}) {
+                    $incourse = 1;
+                    if ($env{'form.forceedit'}) {
+                        $forceview = 1;
+                    } else {
+                        $forceedit = 1;
+                    }
+                    $cfile = $resurl;
+                } elsif ($resurl =~ m{^/adm/wrapper/adm/$cdom/$cnum/\d+/ext\.tool$}) {
+                    $incourse = 1;
+                    if ($env{'form.forceedit'}) {
+                        $forceview = 1;
+                    } else {
+                        $forceedit = 1;
+                    }
+                    $cfile = $resurl;
+                } elsif ($resurl =~ m{^/?adm/viewclasslist$}) {
+                    $incourse = 1;
+                    if ($env{'form.forceedit'}) {
+                        $forceview = 1;
+                    } else {
+                        $forceedit = 1;
+                    }
+                    $cfile = ($resurl =~ m{^/} ? $resurl : "/$resurl");
+                }
+            } elsif ($resurl eq '/res/lib/templates/simpleproblem.problem/smpedit') {
+                my $template = '/res/lib/templates/simpleproblem.problem';
+                if (&is_on_map($template)) { 
+                    $incourse = 1;
+                    $forceview = 1;
+                    $cfile = $template;
+                }
+            } elsif (($resurl =~ m{^/adm/wrapper/ext/}) && ($env{'form.folderpath'} =~ /^supplemental/)) {
+                    $incourse = 1;
+                    if ($env{'form.forceedit'}) {
+                        $forceview = 1;
+                    } else {
+                        $forceedit = 1;
+                    }
+                    $cfile = $resurl;
+            } elsif (($resurl =~ m{^/adm/wrapper/adm/$cdom/$cnum/\d+/ext\.tool$}) && ($env{'form.folderpath'} =~ /^supplemental/)) {
+                $incourse = 1;
+                if ($env{'form.forceedit'}) {
+                    $forceview = 1;
+                } else {
+                    $forceedit = 1;
+                }
+                $cfile = $resurl;
+            } elsif (($resurl eq '/adm/extresedit') && ($symb || $env{'form.folderpath'})) {
+                $incourse = 1;
+                $forceview = 1;
+                if ($symb) {
+                    my ($map,$id,$res)=&decode_symb($symb);
+                    $env{'request.symb'} = $symb;
+                    $cfile = &clutter($res);
+                } else {
+                    $cfile = $env{'form.suppurl'};
+                    my $escfile = &unescape($cfile);
+                    if ($escfile =~ m{^/adm/$cdom/$cnum/\d+/ext\.tool$}) {
+                        $cfile = '/adm/wrapper'.$escfile;
+                    } else {
+                        $escfile =~ s{^http://}{};
+                        $cfile = &escape("/adm/wrapper/ext/$escfile");
+                    }
+                }
+            } elsif ($resurl =~ m{^/?adm/viewclasslist$}) {
+                if ($env{'form.forceedit'}) {
+                    $forceview = 1;
+                } else {
+                    $forceedit = 1;
+                }
+                $cfile = ($resurl =~ m{^/} ? $resurl : "/$resurl");
+            }
+        }
+        if ($uploaded || $incourse) {
+            $home=&homeserver($cnum,$cdom);
+        } elsif ($file !~ m{/$}) {
+            $file=~s{^(priv/$match_domain/$match_username)}{/$1};
+            $file=~s{^($match_domain/$match_username)}{/priv/$1};
+            # Check that the user has permission to edit this resource
+            my $setpriv = 1;
+            my ($cfuname,$cfudom)=&constructaccess($file,$setpriv);
+            if (defined($cfudom)) {
+                $home=&homeserver($cfuname,$cfudom);
+                $cfile=$file;
+            }
+        }
+        if (($cfile ne '') && (!$incourse || $uploaded) && 
+            (($home ne '') && ($home ne 'no_host'))) {
+            my @ids=&current_machine_ids();
+            unless (grep(/^\Q$home\E$/,@ids)) {
+                $switchserver=1;
+            }
+        }
+    }
+    return ($cfile,$home,$switchserver,$forceedit,$forceview);
+}
+
+sub is_course_upload {
+    my ($file,$cnum,$cdom) = @_;
+    my $uploadpath = &LONCAPA::propath($cdom,$cnum);
+    $uploadpath =~ s{^\/}{};
+    if (($file =~ m{^\Q$uploadpath\E/userfiles/(docs|supplemental)/}) ||
+        ($file =~ m{^userfiles/\Q$cdom\E/\Q$cnum\E/(docs|supplemental)/})) {
+        return 1;
+    }
+    return;
+}
+
+sub in_course {
+    my ($udom,$uname,$cdom,$cnum,$type,$hideprivileged) = @_;
+    if ($hideprivileged) {
+        my $skipuser;
+        my %coursehash = &coursedescription($cdom.'_'.$cnum);
+        my @possdoms = ($cdom);  
+        if ($coursehash{'checkforpriv'}) { 
+            push(@possdoms,split(/,/,$coursehash{'checkforpriv'})); 
+        }
+        if (&privileged($uname,$udom,\@possdoms)) {
+            $skipuser = 1;
+            if ($coursehash{'nothideprivileged'}) {
+                foreach my $item (split(/\s*\,\s*/,$coursehash{'nothideprivileged'})) {
+                    my $user;
+                    if ($item =~ /:/) {
+                        $user = $item;
+                    } else {
+                        $user = join(':',split(/[\@]/,$item));
+                    }
+                    if ($user eq $uname.':'.$udom) {
+                        undef($skipuser);
+                        last;
+                    }
+                }
+            }
+            if ($skipuser) {
+                return 0;
+            }
+        }
+    }
+    $type ||= 'any';
+    if (!defined($cdom) || !defined($cnum)) {
+        my $cid  = $env{'request.course.id'};
+        $cdom = $env{'course.'.$cid.'.domain'};
+        $cnum = $env{'course.'.$cid.'.num'};
+    }
+    my $typesref;
+    if (($type eq 'any') || ($type eq 'all')) {
+        $typesref = ['active','previous','future'];
+    } elsif ($type eq 'previous' || $type eq 'future') {
+        $typesref = [$type];
+    }
+    my %roles = &get_my_roles($uname,$udom,'userroles',
+                              $typesref,undef,[$cdom]);
+    my ($tmp) = keys(%roles);
+    return 0 if ($tmp =~ /^(con_lost|error|no_such_host)/i);
+    my @course_roles = grep(/^\Q$cnum\E:\Q$cdom\E:/, keys(%roles));
+    if (@course_roles > 0) {
+        return 1;
+    }
+    return 0;
+}
+
 # --------- File operations in /home/httpd/html/userfiles/$domain/1/2/3/$course
 # input: action, courseID, current domain, intended
 #        path to file, source of file, instruction to parse file for objects,
@@ -2894,7 +3799,9 @@ sub userfileupload {
 					 $codebase,$thumbwidth,$thumbheight,
                                          $resizewidth,$resizeheight,$context,$mimetype);
         } else {
-            $fname=$env{'form.folder'}.'/'.$fname;
+            if ($env{'form.folder'}) {
+                $fname=$env{'form.folder'}.'/'.$fname;
+            }
             return &process_coursefile('uploaddoc',$docuname,$docudom,
 				       $fname,$formname,$parser,
 				       $allfiles,$codebase,$mimetype);
@@ -2909,7 +3816,7 @@ sub userfileupload {
     } else {
         my $docuname=$env{'user.name'};
         my $docudom=$env{'user.domain'};
-        if (exists($env{'form.group'})) {
+        if ((exists($env{'form.group'})) || ($context eq 'syllabus')) {
             $docuname=$env{'course.'.$env{'request.course.id'}.'.num'};
             $docudom=$env{'course.'.$env{'request.course.id'}.'.domain'};
         }
@@ -3032,6 +3939,7 @@ sub finishuserfileupload {
 sub extract_embedded_items {
     my ($fullpath,$allfiles,$codebase,$content) = @_;
     my @state = ();
+    my (%lastids,%related,%shockwave,%flashvars);
     my %javafiles = (
                       codebase => '',
                       code => '',
@@ -3058,13 +3966,35 @@ sub extract_embedded_items {
 		&add_filetype($allfiles,$attr->{'src'},'src');
 	    }
 	    if (lc($tagname) eq 'a') {
-		&add_filetype($allfiles,$attr->{'href'},'href');
+                unless (($attr->{'href'} =~ /^#/) || ($attr->{'href'} eq '')) {
+                    &add_filetype($allfiles,$attr->{'href'},'href');
+                }
 	    }
             if (lc($tagname) eq 'script') {
+                my $src;
                 if ($attr->{'archive'} =~ /\.jar$/i) {
                     &add_filetype($allfiles,$attr->{'archive'},'archive');
                 } else {
-                    &add_filetype($allfiles,$attr->{'src'},'src');
+                    if ($attr->{'src'} ne '') {
+                        $src = $attr->{'src'};
+                        &add_filetype($allfiles,$src,'src');
+                    }
+                }
+                my $text = $p->get_trimmed_text();
+                if ($text =~ /\Qswfobject.registerObject(\E([^\)]+)\)/) {
+                    my @swfargs = split(/,/,$1);
+                    foreach my $item (@swfargs) {
+                        $item =~ s/["']//g;
+                        $item =~ s/^\s+//;
+                        $item =~ s/\s+$//;
+                    }
+                    if (($swfargs[0] ne'') && ($swfargs[2] ne '')) {
+                        if (ref($related{$swfargs[0]}) eq 'ARRAY') {
+                            push(@{$related{$swfargs[0]}},$swfargs[2]);
+                        } else {
+                            $related{$swfargs[0]} = [$swfargs[2]];
+                        }
+                    }
                 }
             }
             if (lc($tagname) eq 'link') {
@@ -3077,6 +4007,9 @@ sub extract_embedded_items {
 		foreach my $item (keys(%javafiles)) {
 		    $javafiles{$item} = '';
 		}
+                if ((lc($tagname) eq 'object') && (lc($state[-2]) ne 'object')) {
+                    $lastids{lc($tagname)} = $attr->{'id'};
+                }
 	    }
 	    if (lc($state[-2]) eq 'object' && lc($tagname) eq 'param') {
 		my $name = lc($attr->{'name'});
@@ -3086,12 +4019,22 @@ sub extract_embedded_items {
 			last;
 		    }
 		}
+                my $pathfrom;
 		foreach my $item (keys(%mediafiles)) {
 		    if ($name eq $item) {
-			&add_filetype($allfiles, $attr->{'value'}, 'value');
+                        $pathfrom = $attr->{'value'};
+                        $shockwave{$lastids{lc($state[-2])}} = $pathfrom;
+			&add_filetype($allfiles,$pathfrom,$name);
 			last;
 		    }
 		}
+                if ($name eq 'flashvars') {
+                    $flashvars{$lastids{lc($state[-2])}} = $attr->{'value'};
+                }
+                if ($pathfrom ne '') {
+                    &embedded_dependency($allfiles,\%related,$lastids{lc($state[-2])},
+                                         $pathfrom);
+                }
 	    }
 	    if (lc($tagname) eq 'embed' || lc($tagname) eq 'applet') {
 		foreach my $item (keys(%javafiles)) {
@@ -3106,7 +4049,34 @@ sub extract_embedded_items {
 			last;
 		    }
 		}
+                if (lc($tagname) eq 'embed') {
+                    if (($attr->{'name'} ne '') && ($attr->{'src'} ne '')) {
+                        &embedded_dependency($allfiles,\%related,$attr->{'name'},
+                                             $attr->{'src'});
+                    }
+                }
 	    }
+            if (lc($tagname) eq 'iframe') {
+                my $src = $attr->{'src'} ;
+                if (($src ne '') && ($src !~ m{^(/|https?://)})) {
+                    &add_filetype($allfiles,$src,'src');
+                } elsif ($src =~ m{^/}) {
+                    if ($env{'request.course.id'}) {
+                        my $cdom = $env{'course.'.$env{'request.course.id'}.'.domain'};
+                        my $cnum = $env{'course.'.$env{'request.course.id'}.'.num'};
+                        my $url = &hreflocation('',$fullpath);
+                        if ($url =~ m{^/uploaded/$cdom/$cnum/docs/(\w+/\d+)/}) {
+                            my $relpath = $1;
+                            if ($src =~ m{^/uploaded/$cdom/$cnum/docs/\Q$relpath\E/(.+)$}) {
+                                &add_filetype($allfiles,$1,'src');
+                            }
+                        }
+                    }
+                }
+            }
+            if ($t->[4] =~ m{/>$}) {
+                pop(@state);
+            }
 	} elsif ($t->[0] eq 'E') {
 	    my ($tagname) = ($t->[1]);
 	    if ($javafiles{'codebase'} ne '') {
@@ -3126,6 +4096,23 @@ sub extract_embedded_items {
 	    pop @state;
 	}
     }
+    foreach my $id (sort(keys(%flashvars))) {
+        if ($shockwave{$id} ne '') {
+            my @pairs = split(/\&/,$flashvars{$id});
+            foreach my $pair (@pairs) {
+                my ($key,$value) = split(/\=/,$pair);
+                if ($key eq 'thumb') {
+                    &add_filetype($allfiles,$value,$key);
+                } elsif ($key eq 'content') {
+                    my ($path) = ($shockwave{$id} =~ m{^(.+/)[^/]+$});
+                    my ($ext) = ($value =~ /\.([^.]+)$/);
+                    if ($ext ne '') {
+                        &add_filetype($allfiles,$path.$value,$ext);
+                    }
+                }
+            }
+        }
+    }
     return 'ok';
 }
 
@@ -3140,6 +4127,21 @@ sub add_filetype {
     }
 }
 
+sub embedded_dependency {
+    my ($allfiles,$related,$identifier,$pathfrom) = @_;
+    if ((ref($allfiles) eq 'HASH') && (ref($related) eq 'HASH')) {
+        if (($identifier ne '') &&
+            (ref($related->{$identifier}) eq 'ARRAY') &&
+            ($pathfrom ne '')) {
+            my ($path) = ($pathfrom =~ m{^(.+/)[^/]+$});
+            foreach my $dep (@{$related->{$identifier}}) {
+                &add_filetype($allfiles,$path.$dep,'object');
+            }
+        }
+    }
+    return;
+}
+
 sub removeuploadedurl {
     my ($url)=@_;	
     my (undef,undef,$udom,$uname,$fname)=split('/',$url,5);    
@@ -3269,6 +4271,7 @@ sub flushcourselogs {
             }
         } else {
             my ($dom,$name) = ($entry=~m{___($match_domain)/($match_name)/(.*)___(\w+)$});
+            if (($dom eq 'uploaded') || ($dom eq 'adm')) { next; }
             my %temphash=($entry => $accesshash{$entry});
             if (&put('nohist_resevaldata',\%temphash,$dom,$name) eq 'ok') {
                 delete $accesshash{$entry};
@@ -3289,7 +4292,7 @@ sub flushcourselogs {
         }
     }
 #
-# Reverse lookup of domain roles (dc, ad, li, sc, au)
+# Reverse lookup of domain roles (dc, ad, li, sc, dh, da, au)
 #
     my %domrolebuffer = ();
     foreach my $entry (keys(%domainrolehash)) {
@@ -3304,10 +4307,19 @@ sub flushcourselogs {
         delete $domainrolehash{$entry};
     }
     foreach my $dom (keys(%domrolebuffer)) {
-	my %servers = &get_servers($dom,'library');
+	my %servers;
+	if (defined(&domain($dom,'primary'))) {
+	    my $primary=&domain($dom,'primary');
+	    my $hostname=&hostname($primary);
+	    $servers{$primary} = $hostname;
+	} else { 
+	    %servers = &get_servers($dom,'library');
+	}
 	foreach my $tryserver (keys(%servers)) {
-	    unless (&reply('domroleput:'.$dom.':'.
-			   $domrolebuffer{$dom},$tryserver) eq 'ok') {
+	    if (&reply('domroleput:'.$dom.':'.
+		       $domrolebuffer{$dom},$tryserver) eq 'ok') {
+		last;
+	    } else {  
 		&logthis('Put of domain roles failed for '.$dom.' and  '.$tryserver);
 	    }
         }
@@ -3397,31 +4409,37 @@ sub linklog {
     $accesshash{$from.'___'.$to.'___comefrom'}=1;
     $accesshash{$to.'___'.$from.'___goto'}=1;
 }
+
+sub statslog {
+    my ($symb,$part,$users,$av_attempts,$degdiff)=@_;
+    if ($users<2) { return; }
+    my %dynstore=&LONCAPA::lonmetadata::dynamic_metadata_storage({
+            'course'       => $env{'request.course.id'},
+            'sections'     => '"all"',
+            'num_students' => $users,
+            'part'         => $part,
+            'symb'         => $symb,
+            'mean_tries'   => $av_attempts,
+            'deg_of_diff'  => $degdiff});
+    foreach my $key (keys(%dynstore)) {
+        $accesshash{$key}=$dynstore{$key};
+    }
+}
   
 sub userrolelog {
     my ($trole,$username,$domain,$area,$tstart,$tend)=@_;
-    if (($trole=~/^ca/) || ($trole=~/^aa/) ||
-        ($trole=~/^in/) || ($trole=~/^cc/) ||
-        ($trole=~/^ep/) || ($trole=~/^cr/) ||
-        ($trole=~/^ta/) || ($trole=~/^co/)) {
+    if ( $trole =~ /^(ca|aa|in|cc|ep|cr|ta|co)/ ) {
        my (undef,$rudom,$runame,$rsec)=split(/\//,$area);
        $userrolehash
          {$trole.':'.$username.':'.$domain.':'.$runame.':'.$rudom.':'.$rsec}
                     =$tend.':'.$tstart;
     }
-    if (($env{'request.role'} =~ /dc\./) &&
-	(($trole=~/^au/) || ($trole=~/^in/) ||
-	 ($trole=~/^cc/) || ($trole=~/^ep/) ||
-	 ($trole=~/^cr/) || ($trole=~/^ta/) ||
-         ($trole=~/^co/))) {
+    if ($env{'request.role'} =~ /dc\./ && $trole =~ /^(au|in|cc|ep|cr|ta|co)/) {
        $userrolehash
          {$trole.':'.$username.':'.$domain.':'.$env{'user.name'}.':'.$env{'user.domain'}.':'}
                     =$tend.':'.$tstart;
     }
-    if (($trole=~/^dc/) || ($trole=~/^ad/) ||
-        ($trole=~/^li/) || ($trole=~/^li/) ||
-        ($trole=~/^au/) || ($trole=~/^dg/) ||
-        ($trole=~/^sc/)) {
+    if ($trole =~ /^(dc|ad|li|au|dg|sc|dh|da)/ ) {
        my (undef,$rudom,$runame,$rsec)=split(/\//,$area);
        $domainrolehash
          {$trole.':'.$username.':'.$domain.':'.$runame.':'.$rudom.':'.$rsec}
@@ -3431,38 +4449,70 @@ sub userrolelog {
 
 sub courserolelog {
     my ($trole,$username,$domain,$area,$tstart,$tend,$delflag,$selfenroll,$context)=@_;
-    if (($trole eq 'cc') || ($trole eq 'in') ||
-        ($trole eq 'ep') || ($trole eq 'ad') ||
-        ($trole eq 'ta') || ($trole eq 'st') ||
-        ($trole=~/^cr/) || ($trole eq 'gr') ||
-        ($trole eq 'co')) {
-        if ($area =~ m-^/($match_domain)/($match_courseid)/?([^/]*)-) {
-            my $cdom = $1;
-            my $cnum = $2;
-            my $sec = $3;
-            my $namespace = 'rolelog';
-            my %storehash = (
-                               role    => $trole,
-                               start   => $tstart,
-                               end     => $tend,
-                               selfenroll => $selfenroll,
-                               context    => $context,
-                            );
-            if ($trole eq 'gr') {
-                $namespace = 'groupslog';
-                $storehash{'group'} = $sec;
-            } else {
-                $storehash{'section'} = $sec;
-            }
-            &instructor_log($namespace,\%storehash,$delflag,$username,$domain,$cnum,$cdom);
-            if (($trole ne 'st') || ($sec ne '')) {
-                &devalidate_cache_new('getcourseroles',$cdom.'_'.$cnum);
-            }
+    if ($area =~ m-^/($match_domain)/($match_courseid)/?([^/]*)-) {
+        my $cdom = $1;
+        my $cnum = $2;
+        my $sec = $3;
+        my $namespace = 'rolelog';
+        my %storehash = (
+                           role    => $trole,
+                           start   => $tstart,
+                           end     => $tend,
+                           selfenroll => $selfenroll,
+                           context    => $context,
+                        );
+        if ($trole eq 'gr') {
+            $namespace = 'groupslog';
+            $storehash{'group'} = $sec;
+        } else {
+            $storehash{'section'} = $sec;
+        }
+        &write_log('course',$namespace,\%storehash,$delflag,$username,
+                   $domain,$cnum,$cdom);
+        if (($trole ne 'st') || ($sec ne '')) {
+            &devalidate_cache_new('getcourseroles',$cdom.'_'.$cnum);
         }
     }
     return;
 }
 
+sub domainrolelog {
+    my ($trole,$username,$domain,$area,$tstart,$tend,$delflag,$context)=@_;
+    if ($area =~ m{^/($match_domain)/$}) {
+        my $cdom = $1;
+        my $domconfiguser = &Apache::lonnet::get_domainconfiguser($cdom);
+        my $namespace = 'rolelog';
+        my %storehash = (
+                           role    => $trole,
+                           start   => $tstart,
+                           end     => $tend,
+                           context => $context,
+                        );
+        &write_log('domain',$namespace,\%storehash,$delflag,$username,
+                   $domain,$domconfiguser,$cdom);
+    }
+    return;
+
+}
+
+sub coauthorrolelog {
+    my ($trole,$username,$domain,$area,$tstart,$tend,$delflag,$context)=@_;
+    if ($area =~ m{^/($match_domain)/($match_username)$}) {
+        my $audom = $1;
+        my $auname = $2;
+        my $namespace = 'rolelog';
+        my %storehash = (
+                           role    => $trole,
+                           start   => $tstart,
+                           end     => $tend,
+                           context => $context,
+                        );
+        &write_log('author',$namespace,\%storehash,$delflag,$username,
+                   $domain,$auname,$audom);
+    }
+    return;
+}
+
 sub get_course_adv_roles {
     my ($cid,$codes) = @_;
     $cid=$env{'request.course.id'} unless (defined($cid));
@@ -3476,6 +4526,10 @@ sub get_course_adv_roles {
             $nothide{$user}=1;
         }
     }
+    my @possdoms = ($coursehash{'domain'});
+    if ($coursehash{'checkforpriv'}) {
+        push(@possdoms,split(/,/,$coursehash{'checkforpriv'}));
+    }
     my %returnhash=();
     my %dumphash=
             &dump('nohist_userroles',$coursehash{'domain'},$coursehash{'num'});
@@ -3488,20 +4542,7 @@ sub get_course_adv_roles {
         if (($tstart) && ($now<$tstart)) { next; }
         my ($role,$username,$domain,$section)=split(/\:/,$entry);
 	if ($username eq '' || $domain eq '') { next; }
-        unless (ref($privileged{$domain}) eq 'HASH') {
-            my %dompersonnel =
-                &Apache::lonnet::get_domain_roles($domain,['dc'],$now,$now);
-            $privileged{$domain} = {};
-            foreach my $server (keys(%dompersonnel)) {
-                if (ref($dompersonnel{$server}) eq 'HASH') {
-                    foreach my $user (keys(%{$dompersonnel{$server}})) {
-                        my ($trole,$uname,$udom) = split(/:/,$user);
-                        $privileged{$udom}{$uname} = 1;
-                    }
-                }
-            }
-        }
-        if ((exists($privileged{$domain}{$username})) && 
+        if ((&privileged($username,$domain,\@possdoms)) &&
             (!$nothide{$username.':'.$domain})) { next; }
 	if ($role eq 'cr') { next; }
         if ($codes) {
@@ -3530,11 +4571,9 @@ sub get_my_roles {
     unless (defined($udom)) { $udom=$env{'user.domain'}; }
     my (%dumphash,%nothide);
     if ($context eq 'userroles') {
-        my $extra = &freeze_escape({'skipcheck' => 1});
-        %dumphash = &dump('roles',$udom,$uname,'.',undef,$extra);
+        %dumphash = &dump('roles',$udom,$uname);
     } else {
-        %dumphash=
-            &dump('nohist_userroles',$udom,$uname);
+        %dumphash = &dump('nohist_userroles',$udom,$uname);
         if ($hidepriv) {
             my %coursehash=&coursedescription($udom.'_'.$uname);
             foreach my $user (split(/\s*\,\s*/,$coursehash{'nothideprivileged'})) {
@@ -3576,7 +4615,7 @@ sub get_my_roles {
         }
         my ($rolecode,$username,$domain,$section,$area);
         if ($context eq 'userroles') {
-            ($area,$rolecode) = split(/_/,$entry);
+            ($area,$rolecode) = ($entry =~ /^(.+)_([^_]+)$/);
             (undef,$domain,$username,$section) = split(/\//,$area);
         } else {
             ($role,$username,$domain,$section) = split(/\:/,$entry);
@@ -3602,28 +4641,15 @@ sub get_my_roles {
             }
         }
         if ($hidepriv) {
+            my @privroles = ('dc','su');
             if ($context eq 'userroles') {
-                if ((&privileged($username,$domain)) &&
-                    (!$nothide{$username.':'.$domain})) {
-                    next;
-                }
+                next if (grep(/^\Q$role\E$/,@privroles));
             } else {
-                unless (ref($privileged{$domain}) eq 'HASH') {
-                    my %dompersonnel =
-                        &Apache::lonnet::get_domain_roles($domain,['dc'],$now,$now);
-                    $privileged{$domain} = {};
-                    if (keys(%dompersonnel)) {
-                        foreach my $server (keys(%dompersonnel)) {
-                            if (ref($dompersonnel{$server}) eq 'HASH') {
-                                foreach my $user (keys(%{$dompersonnel{$server}})) {
-                                    my ($trole,$uname,$udom) = split(/:/,$user);
-                                    $privileged{$udom}{$uname} = $trole;
-                                }
-                            }
-                        }
-                    }
+                my $possdoms = [$domain];
+                if (ref($roledoms) eq 'ARRAY') {
+                   push(@{$possdoms},@{$roledoms}); 
                 }
-                if (exists($privileged{$domain}{$username})) {
+                if (&privileged($username,$domain,$possdoms,\@privroles)) {
                     if (!$nothide{$username.':'.$domain}) {
                         next;
                     }
@@ -3640,6 +4666,195 @@ sub get_my_roles {
     return %returnhash;
 }
 
+sub get_all_adhocroles {
+    my ($dom) = @_;
+    my @roles_by_num = ();
+    my %domdefaults = &get_domain_defaults($dom);
+    my (%description,%access_in_dom,%access_info);
+    if (ref($domdefaults{'adhocroles'}) eq 'HASH') {
+        my $count = 0;
+        my %domcurrent = %{$domdefaults{'adhocroles'}};
+        my %ordered;
+        foreach my $role (sort(keys(%domcurrent))) {
+            my ($order,$desc,$access_in_dom);
+            if (ref($domcurrent{$role}) eq 'HASH') {
+                $order = $domcurrent{$role}{'order'};
+                $desc = $domcurrent{$role}{'desc'};
+                $access_in_dom{$role} = $domcurrent{$role}{'access'};
+                $access_info{$role} = $domcurrent{$role}{$access_in_dom{$role}};
+            }
+            if ($order eq '') {
+                $order = $count;
+            }
+            $ordered{$order} = $role;
+            if ($desc ne '') {
+                $description{$role} = $desc;
+            } else {
+                $description{$role}= $role;
+            }
+            $count++;
+        }
+        foreach my $item (sort {$a <=> $b } (keys(%ordered))) {
+            push(@roles_by_num,$ordered{$item});
+        }
+    }
+    return (\@roles_by_num,\%description,\%access_in_dom,\%access_info);
+}
+
+sub get_my_adhocroles {
+    my ($cid,$checkreg) = @_;
+    my ($cdom,$cnum,%info,@possroles,$description,$roles_by_num);
+    if ($env{'request.course.id'} eq $cid) {
+        $cdom = $env{'course.'.$cid.'.domain'};
+        $cnum = $env{'course.'.$cid.'.num'};
+        $info{'internal.coursecode'} = $env{'course.'.$cid.'.internal.coursecode'};
+    } elsif ($cid =~ /^($match_domain)_($match_courseid)$/) {
+        $cdom = $1;
+        $cnum = $2;
+        %info = &Apache::lonnet::get('environment',['internal.coursecode'],
+                                     $cdom,$cnum);
+    }
+    if (($info{'internal.coursecode'} ne '') && ($checkreg)) {
+        my $user = $env{'user.name'}.':'.$env{'user.domain'};
+        my %rosterhash = &get('classlist',[$user],$cdom,$cnum);
+        if ($rosterhash{$user} ne '') {
+            my $type = (split(/:/,$rosterhash{$user}))[5];
+            return ([],{}) if ($type eq 'auto');
+        }
+    }
+    if (($cdom ne '') && ($cnum ne ''))  {
+        if (($env{"user.role.dh./$cdom/"}) || ($env{"user.role.da./$cdom/"})) {
+            my $then=$env{'user.login.time'};
+            my $update=$env{'user.update.time'};
+            if (!$update) {
+                $update = $then;
+            }
+            my @liveroles;
+            foreach my $role ('dh','da') {
+                if ($env{"user.role.$role./$cdom/"}) {
+                    my ($tstart,$tend)=split(/\./,$env{"user.role.$role./$cdom/"});
+                    my $limit = $update;
+                    if ($env{'request.role'} eq "$role./$cdom/") {
+                        $limit = $then;
+                    }
+                    my $activerole = 1;
+                    if ($tstart && $tstart>$limit) { $activerole = 0; }
+                    if ($tend   && $tend  <$limit) { $activerole = 0; }
+                    if ($activerole) {
+                        push(@liveroles,$role);
+                    }
+                }
+            }
+            if (@liveroles) {
+                if (&homeserver($cnum,$cdom) ne 'no_host') {
+                    my ($accessref,$accessinfo,%access_in_dom);
+                    ($roles_by_num,$description,$accessref,$accessinfo) = &get_all_adhocroles($cdom);
+                    if (ref($roles_by_num) eq 'ARRAY') {
+                        if (@{$roles_by_num}) {
+                            my %settings;
+                            if ($env{'request.course.id'} eq $cid) {
+                                foreach my $envkey (keys(%env)) {
+                                    if ($envkey =~ /^\Qcourse.$cid.\E(internal\.adhoc.+)$/) {
+                                        $settings{$1} = $env{$envkey};
+                                    }
+                                }
+                            } else {
+                                %settings = &dump('environment',$cdom,$cnum,'internal\.adhoc');
+                            }
+                            my %setincrs;
+                            if ($settings{'internal.adhocaccess'}) {
+                                map { $setincrs{$_} = 1; } split(/,/,$settings{'internal.adhocaccess'});
+                            }
+                            my @statuses;
+                            if ($env{'environment.inststatus'}) {
+                                @statuses = split(/,/,$env{'environment.inststatus'});
+                            }
+                            my $user = $env{'user.name'}.':'.$env{'user.domain'};
+                            if (ref($accessref) eq 'HASH') {
+                                %access_in_dom = %{$accessref};
+                            }
+                            foreach my $role (@{$roles_by_num}) {
+                                my ($curraccess,@okstatus,@personnel);
+                                if ($setincrs{$role}) {
+                                    ($curraccess,my $rest) = split(/=/,$settings{'internal.adhoc.'.$role});
+                                    if ($curraccess eq 'status') {
+                                        @okstatus = split(/\&/,$rest);
+                                    } elsif (($curraccess eq 'exc') || ($curraccess eq 'inc')) {
+                                        @personnel = split(/\&/,$rest);
+                                    }
+                                } else {
+                                    $curraccess = $access_in_dom{$role};
+                                    if (ref($accessinfo) eq 'HASH') {
+                                        if ($curraccess eq 'status') {
+                                            if (ref($accessinfo->{$role}) eq 'ARRAY') {
+                                                @okstatus = @{$accessinfo->{$role}};
+                                            }
+                                        } elsif (($curraccess eq 'exc') || ($curraccess eq 'inc')) {
+                                            if (ref($accessinfo->{$role}) eq 'ARRAY') {
+                                                @personnel = @{$accessinfo->{$role}};
+                                            }
+                                        }
+                                    }
+                                }
+                                if ($curraccess eq 'none') {
+                                    next;
+                                } elsif ($curraccess eq 'all') {
+                                    push(@possroles,$role);
+                                } elsif ($curraccess eq 'dh') {
+                                    if (grep(/^dh$/,@liveroles)) {
+                                        push(@possroles,$role);
+                                    } else {
+                                        next;
+                                    }
+                                } elsif ($curraccess eq 'da') {
+                                    if (grep(/^da$/,@liveroles)) {
+                                        push(@possroles,$role);
+                                    } else {
+                                        next;
+                                    }
+                                } elsif ($curraccess eq 'status') {
+                                    if (@okstatus) {
+                                        if (!@statuses) {
+                                            if (grep(/^default$/,@okstatus)) {
+                                                push(@possroles,$role);
+                                            }
+                                        } else {
+                                            foreach my $status (@okstatus) {
+                                                if (grep(/^\Q$status\E$/,@statuses)) {
+                                                    push(@possroles,$role);
+                                                    last;
+                                                }
+                                            }
+                                        }
+                                    }
+                                } elsif (($curraccess eq 'exc') || ($curraccess eq 'inc')) {
+                                    if (grep(/^\Q$user\E$/,@personnel)) {
+                                        if ($curraccess eq 'exc') {
+                                            push(@possroles,$role);
+                                        }
+                                    } elsif ($curraccess eq 'inc') {
+                                        push(@possroles,$role);
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    unless (ref($description) eq 'HASH') {
+        if (ref($roles_by_num) eq 'ARRAY') {
+            my %desc;
+            map { $desc{$_} = $_; } (@{$roles_by_num});
+            $description = \%desc;
+        } else {
+            $description = {};
+        }
+    }
+    return (\@possroles,$description);
+}
+
 # ----------------------------------------------------- Frontpage Announcements
 #
 #
@@ -3715,7 +4930,8 @@ sub courseiddump {
     my ($domfilter,$descfilter,$sincefilter,$instcodefilter,$ownerfilter,
         $coursefilter,$hostidflag,$hostidref,$typefilter,$regexp_ok,
         $selfenrollonly,$catfilter,$showhidden,$caller,$cloner,$cc_clone,
-        $cloneonly,$createdbefore,$createdafter,$creationcontext,$domcloner)=@_;
+        $cloneonly,$createdbefore,$createdafter,$creationcontext,$domcloner,
+        $hasuniquecode,$reqcrsdom,$reqinstcode)=@_;
     my $as_hash = 1;
     my %returnhash;
     if (!$domfilter) { $domfilter=''; }
@@ -3727,18 +4943,33 @@ sub courseiddump {
 
 	    if (($domfilter eq '') ||
 		(&host_domain($tryserver) eq $domfilter)) {
-                my $rep = 
-                  &reply('courseiddump:'.&host_domain($tryserver).':'.
-                         $sincefilter.':'.&escape($descfilter).':'.
-                         &escape($instcodefilter).':'.&escape($ownerfilter).
-                         ':'.&escape($coursefilter).':'.&escape($typefilter).
-                         ':'.&escape($regexp_ok).':'.$as_hash.':'.
-                         &escape($selfenrollonly).':'.&escape($catfilter).':'.
-                         $showhidden.':'.$caller.':'.&escape($cloner).':'.
-                         &escape($cc_clone).':'.$cloneonly.':'.
-                         &escape($createdbefore).':'.&escape($createdafter).':'.
-                         &escape($creationcontext).':'.$domcloner,
-                         $tryserver);
+                my $rep;
+                if (grep { $_ eq $tryserver } current_machine_ids()) {
+                    $rep = LONCAPA::Lond::dump_course_id_handler(
+                        join(":", (&host_domain($tryserver), $sincefilter, 
+                                &escape($descfilter), &escape($instcodefilter), 
+                                &escape($ownerfilter), &escape($coursefilter),
+                                &escape($typefilter), &escape($regexp_ok), 
+                                $as_hash, &escape($selfenrollonly), 
+                                &escape($catfilter), $showhidden, $caller, 
+                                &escape($cloner), &escape($cc_clone), $cloneonly, 
+                                &escape($createdbefore), &escape($createdafter), 
+                                &escape($creationcontext),$domcloner,$hasuniquecode,
+                                $reqcrsdom,&escape($reqinstcode))));
+                } else {
+                    $rep = &reply('courseiddump:'.&host_domain($tryserver).':'.
+                             $sincefilter.':'.&escape($descfilter).':'.
+                             &escape($instcodefilter).':'.&escape($ownerfilter).
+                             ':'.&escape($coursefilter).':'.&escape($typefilter).
+                             ':'.&escape($regexp_ok).':'.$as_hash.':'.
+                             &escape($selfenrollonly).':'.&escape($catfilter).':'.
+                             $showhidden.':'.$caller.':'.&escape($cloner).':'.
+                             &escape($cc_clone).':'.$cloneonly.':'.
+                             &escape($createdbefore).':'.&escape($createdafter).':'.
+                             &escape($creationcontext).':'.$domcloner.':'.$hasuniquecode.
+                             ':'.$reqcrsdom.':'.&escape($reqinstcode),$tryserver);
+                }
+                     
                 my @pairs=split(/\&/,$rep);
                 foreach my $item (@pairs) {
                     my ($key,$value)=split(/\=/,$item,2);
@@ -3844,7 +5075,7 @@ sub get_domain_roles {
     }
     my $rolelist;
     if (ref($roles) eq 'ARRAY') {
-        $rolelist = join(':',@{$roles});
+        $rolelist = join('&',@{$roles});
     }
     my %personnel = ();
 
@@ -3864,6 +5095,21 @@ sub get_domain_roles {
     return %personnel;
 }
 
+sub get_active_domroles {
+    my ($dom,$roles) = @_;
+    return () unless (ref($roles) eq 'ARRAY');
+    my $now = time;
+    my %dompersonnel = &get_domain_roles($dom,$roles,$now,$now);
+    my %domroles;
+    foreach my $server (keys(%dompersonnel)) {
+        foreach my $user (sort(keys(%{$dompersonnel{$server}}))) {
+            my ($trole,$uname,$udom,$runame,$rudom,$rsec) = split(/:/,$user);
+            $domroles{$uname.':'.$udom} = $dompersonnel{$server}{$user};
+        }
+    }
+    return %domroles;
+}
+
 # ----------------------------------------------------------- Interval timing 
 
 {
@@ -3875,12 +5121,13 @@ my $cachedkey='';
 # The cached times for this user
 my %cachedtimes=();
 # When this was last done
-my $cachedtime=();
+my $cachedtime='';
 
 sub load_all_first_access {
-    my ($uname,$udom)=@_;
+    my ($uname,$udom,$ignorecache)=@_;
     if (($cachedkey eq $uname.':'.$udom) &&
-        (abs($cachedtime-time)<5)) {
+        (abs($cachedtime-time)<5) && (!$env{'form.markaccess'}) &&
+        (!$ignorecache)) {
         return;
     }
     $cachedtime=time;
@@ -3889,10 +5136,11 @@ sub load_all_first_access {
 }
 
 sub get_first_access {
-    my ($type,$argsymb)=@_;
+    my ($type,$argsymb,$argmap,$ignorecache)=@_;
     my ($symb,$courseid,$udom,$uname)=&whichuser();
     if ($argsymb) { $symb=$argsymb; }
     my ($map,$id,$res)=&decode_symb($symb);
+    if ($argmap) { $map = $argmap; }
     if ($type eq 'course') {
 	$res='course';
     } elsif ($type eq 'map') {
@@ -3900,12 +5148,12 @@ sub get_first_access {
     } else {
 	$res=$symb;
     }
-    &load_all_first_access($uname,$udom);
+    &load_all_first_access($uname,$udom,$ignorecache);
     return $cachedtimes{"$courseid\0$res"};
 }
 
 sub set_first_access {
-    my ($type)=@_;
+    my ($type,$interval)=@_;
     my ($symb,$courseid,$udom,$uname)=&whichuser();
     my ($map,$id,$res)=&decode_symb($symb);
     if ($type eq 'course') {
@@ -3916,13 +5164,27 @@ sub set_first_access {
 	$res=$symb;
     }
     $cachedkey='';
-    my $firstaccess=&get_first_access($type,$symb);
+    my $firstaccess=&get_first_access($type,$symb,$map);
     if (!$firstaccess) {
-	return &put('firstaccesstimes',{"$courseid\0$res"=>time},$udom,$uname);
+        my $start = time;
+	my $putres = &put('firstaccesstimes',{"$courseid\0$res"=>$start},
+                          $udom,$uname);
+        if ($putres eq 'ok') {
+            &put('timerinterval',{"$courseid\0$res"=>$interval},
+                 $udom,$uname); 
+            &appenv(
+                     {
+                        'course.'.$courseid.'.firstaccess.'.$res   => $start,
+                        'course.'.$courseid.'.timerinterval.'.$res => $interval,
+                     }
+                  );
+        }
+        return $putres;
     }
     return 'already_set';
 }
 }
+
 # --------------------------------------------- Set Expire Date for Spreadsheet
 
 sub expirespread {
@@ -4293,7 +5555,7 @@ sub tmprestore {
 # ----------------------------------------------------------------------- Store
 
 sub store {
-    my ($storehash,$symb,$namespace,$domain,$stuname) = @_;
+    my ($storehash,$symb,$namespace,$domain,$stuname,$laststore) = @_;
     my $home='';
 
     if ($stuname) { $home=&homeserver($stuname,$domain); }
@@ -4323,13 +5585,13 @@ sub store {
     }
     $namevalue=~s/\&$//;
     &courselog($symb.':'.$stuname.':'.$domain.':STORE:'.$namevalue);
-    return reply("store:$domain:$stuname:$namespace:$symb:$namevalue","$home");
+    return reply("store:$domain:$stuname:$namespace:$symb:$namevalue:$laststore","$home");
 }
 
 # -------------------------------------------------------------- Critical Store
 
 sub cstore {
-    my ($storehash,$symb,$namespace,$domain,$stuname) = @_;
+    my ($storehash,$symb,$namespace,$domain,$stuname,$laststore) = @_;
     my $home='';
 
     if ($stuname) { $home=&homeserver($stuname,$domain); }
@@ -4360,7 +5622,7 @@ sub cstore {
     $namevalue=~s/\&$//;
     &courselog($symb.':'.$stuname.':'.$domain.':CSTORE:'.$namevalue);
     return critical
-                ("store:$domain:$stuname:$namespace:$symb:$namevalue","$home");
+                ("store:$domain:$stuname:$namespace:$symb:$namevalue:$laststore","$home");
 }
 
 # --------------------------------------------------------------------- Restore
@@ -4372,9 +5634,12 @@ sub restore {
     if ($stuname) { $home=&homeserver($stuname,$domain); }
 
     if (!$symb) {
-      unless ($symb=escape(&symbread())) { return ''; }
+        return if ($namespace eq 'courserequests');
+        unless ($symb=escape(&symbread())) { return ''; }
     } else {
-      $symb=&escape(&symbclean($symb));
+        unless ($namespace eq 'courserequests') {
+            $symb=&escape(&symbclean($symb));
+        }
     }
     if (!$namespace) { 
        unless ($namespace=$env{'request.course.id'}) { 
@@ -4509,109 +5774,217 @@ sub update_released_required {
 # -------------------------------------------------See if a user is privileged
 
 sub privileged {
-    my ($username,$domain)=@_;
-    my $rolesdump=&reply("dump:$domain:$username:roles",
-			&homeserver($username,$domain));
-    if (($rolesdump eq 'con_lost') || ($rolesdump eq '') || 
-        ($rolesdump =~ /^error:/)) {
-        return 0;
-    }
-    my $now=time;
-    if ($rolesdump ne '') {
-        foreach my $entry (split(/&/,$rolesdump)) {
-	    if ($entry!~/^rolesdef_/) {
-		my ($area,$role)=split(/=/,$entry);
-		$area=~s/\_\w\w$//;
-		my ($trole,$tend,$tstart)=split(/_/,$role);
-		if (($trole eq 'dc') || ($trole eq 'su')) {
-		    my $active=1;
-		    if ($tend) {
-			if ($tend<$now) { $active=0; }
-		    }
-		    if ($tstart) {
-			if ($tstart>$now) { $active=0; }
-		    }
-		    if ($active) { return 1; }
-		}
-	    }
-	}
+    my ($username,$domain,$possdomains,$possroles)=@_;
+    my $now = time;
+    my $roles;
+    if (ref($possroles) eq 'ARRAY') {
+        $roles = $possroles; 
+    } else {
+        $roles = ['dc','su'];
+    }
+    if (ref($possdomains) eq 'ARRAY') {
+        my %privileged = &privileged_by_domain($possdomains,$roles);
+        foreach my $dom (@{$possdomains}) {
+            if (($username =~ /^$match_username$/) && ($domain =~ /^$match_domain$/) &&
+                (ref($privileged{$dom}) eq 'HASH')) {
+                foreach my $role (@{$roles}) {
+                    if (ref($privileged{$dom}{$role}) eq 'HASH') {
+                        if (exists($privileged{$dom}{$role}{$username.':'.$domain})) {
+                            my ($end,$start) = split(/:/,$privileged{$dom}{$role}{$username.':'.$domain});
+                            return 1 unless (($end && $end < $now) ||
+                                             ($start && $start > $now));
+                        }
+                    }
+                }
+            }
+        }
+    } else {
+        my %rolesdump = &dump("roles", $domain, $username) or return 0;
+        my $now = time;
+
+        for my $role (@rolesdump{grep { ! /^rolesdef_/ } keys(%rolesdump)}) {
+            my ($trole, $tend, $tstart) = split(/_/, $role);
+            if (grep(/^\Q$trole\E$/,@{$roles})) {
+                return 1 unless ($tend && $tend < $now) 
+                        or ($tstart && $tstart > $now);
+            }
+        }
     }
     return 0;
 }
 
+sub privileged_by_domain {
+    my ($domains,$roles) = @_;
+    my %privileged = ();
+    my $cachetime = 60*60*24;
+    my $now = time;
+    unless ((ref($domains) eq 'ARRAY') && (ref($roles) eq 'ARRAY')) {
+        return %privileged;
+    }
+    foreach my $dom (@{$domains}) {
+        next if (ref($privileged{$dom}) eq 'HASH');
+        my $needroles;
+        foreach my $role (@{$roles}) {
+            my ($result,$cached)=&is_cached_new('priv_'.$role,$dom);
+            if (defined($cached)) {
+                if (ref($result) eq 'HASH') {
+                    $privileged{$dom}{$role} = $result;
+                }
+            } else {
+                $needroles = 1;
+            }
+        }
+        if ($needroles) {
+            my %dompersonnel = &get_domain_roles($dom,$roles);
+            $privileged{$dom} = {};
+            foreach my $server (keys(%dompersonnel)) {
+                if (ref($dompersonnel{$server}) eq 'HASH') {
+                    foreach my $item (keys(%{$dompersonnel{$server}})) {
+                        my ($trole,$uname,$udom,$rest) = split(/:/,$item,4);
+                        my ($end,$start) = split(/:/,$dompersonnel{$server}{$item});
+                        next if ($end && $end < $now);
+                        $privileged{$dom}{$trole}{$uname.':'.$udom} = 
+                            $dompersonnel{$server}{$item};
+                    }
+                }
+            }
+            if (ref($privileged{$dom}) eq 'HASH') {
+                foreach my $role (@{$roles}) {
+                    if (ref($privileged{$dom}{$role}) eq 'HASH') {
+                        &do_cache_new('priv_'.$role,$dom,$privileged{$dom}{$role},$cachetime);
+                    } else {
+                        my %hash = ();
+                        &do_cache_new('priv_'.$role,$dom,\%hash,$cachetime);
+                    }
+                }
+            }
+        }
+    }
+    return %privileged;
+}
+
 # -------------------------------------------------------- Get user privileges
 
 sub rolesinit {
-    my ($domain,$username,$authhost)=@_;
-    my $now=time;
-    my %userroles = ('user.login.time' => $now);
-    my $extra = &freeze_escape({'skipcheck' => 1});
-    my $rolesdump=reply("dump:$domain:$username:roles:.::$extra",$authhost);
-    if (($rolesdump eq 'con_lost') || ($rolesdump eq '') || 
-        ($rolesdump =~ /^error:/)) {
-        return \%userroles;
+    my ($domain, $username) = @_;
+    my %userroles = ('user.login.time' => time);
+    my %rolesdump = &dump("roles", $domain, $username) or return \%userroles;
+
+    # firstaccess and timerinterval are related to timed maps/resources. 
+    # also, blocking can be triggered by an activating timer
+    # it's saved in the user's %env.
+    my %firstaccess = &dump('firstaccesstimes', $domain, $username);
+    my %timerinterval = &dump('timerinterval', $domain, $username);
+    my (%coursetimerstarts, %firstaccchk, %firstaccenv, %coursetimerintervals,
+        %timerintchk, %timerintenv);
+
+    foreach my $key (keys(%firstaccess)) {
+        my ($cid, $rest) = split(/\0/, $key);
+        $coursetimerstarts{$cid}{$rest} = $firstaccess{$key};
+    }
+
+    foreach my $key (keys(%timerinterval)) {
+        my ($cid,$rest) = split(/\0/,$key);
+        $coursetimerintervals{$cid}{$rest} = $timerinterval{$key};
     }
+
     my %allroles=();
-    my %allgroups=();   
+    my %allgroups=();
 
-    if ($rolesdump ne '') {
-        foreach my $entry (split(/&/,$rolesdump)) {
-	  if ($entry!~/^rolesdef_/) {
-            my ($area,$role)=split(/=/,$entry);
-	    $area=~s/\_\w\w$//;
-            my ($trole,$tend,$tstart,$group_privs);
-	    if ($role=~/^cr/) { 
-		if ($role=~m|^(cr/$match_domain/$match_username/[a-zA-Z0-9]+)_(.*)$|) {
-		    ($trole,my $trest)=($role=~m|^(cr/$match_domain/$match_username/[a-zA-Z0-9]+)_(.*)$|);
-		    ($tend,$tstart)=split('_',$trest);
-		} else {
-		    $trole=$role;
-		}
-            } elsif ($role =~ m|^gr/|) {
-                ($trole,$tend,$tstart) = split(/_/,$role);
-                next if ($tstart eq '-1');
-                ($trole,$group_privs) = split(/\//,$trole);
-                $group_privs = &unescape($group_privs);
-	    } else {
-		($trole,$tend,$tstart)=split(/_/,$role);
-	    }
-	    my %new_role = &set_arearole($trole,$area,$tstart,$tend,$domain,
-					 $username);
-	    @userroles{keys(%new_role)} = @new_role{keys(%new_role)};
-            if (($tend!=0) && ($tend<$now)) { $trole=''; }
-            if (($tstart!=0) && ($tstart>$now)) { $trole=''; }
-            if (($area ne '') && ($trole ne '')) {
-		my $spec=$trole.'.'.$area;
-		my ($tdummy,$tdomain,$trest)=split(/\//,$area);
-		if ($trole =~ /^cr\//) {
-                    &custom_roleprivs(\%allroles,$trole,$tdomain,$trest,$spec,$area);
-                } elsif ($trole eq 'gr') {
-                    &group_roleprivs(\%allgroups,$area,$group_privs,$tend,$tstart);
-		} else {
-                    &standard_roleprivs(\%allroles,$trole,$tdomain,$spec,$trest,$area);
-		}
+    for my $area (grep { ! /^rolesdef_/ } keys(%rolesdump)) {
+        my $role = $rolesdump{$area};
+        $area =~ s/\_\w\w$//;
+
+        my ($trole, $tend, $tstart, $group_privs);
+
+        if ($role =~ /^cr/) {
+        # Custom role, defined by a user 
+        # e.g., user.role.cr/msu/smith/mynewrole
+            if ($role =~ m|^(cr/$match_domain/$match_username/[a-zA-Z0-9]+)_(.*)$|) {
+                $trole = $1;
+                ($tend, $tstart) = split('_', $2);
+            } else {
+                $trole = $role;
             }
-          }
+        } elsif ($role =~ m|^gr/|) {
+        # Role of member in a group, defined within a course/community
+        # e.g., user.role.gr/msu/04935610a19ee4a5fmsul1/leopards
+            ($trole, $tend, $tstart) = split(/_/, $role);
+            next if $tstart eq '-1';
+            ($trole, $group_privs) = split(/\//, $trole);
+            $group_privs = &unescape($group_privs);
+        } else {
+        # Just a normal role, defined in roles.tab
+            ($trole, $tend, $tstart) = split(/_/,$role);
+        }
+
+        my %new_role = &set_arearole($trole,$area,$tstart,$tend,$domain,
+                 $username);
+        @userroles{keys(%new_role)} = @new_role{keys(%new_role)};
+
+        # role expired or not available yet?
+        $trole = '' if ($tend != 0 && $tend < $userroles{'user.login.time'}) or 
+            ($tstart != 0 && $tstart > $userroles{'user.login.time'});
+
+        next if $area eq '' or $trole eq '';
+
+        my $spec = "$trole.$area";
+        my ($tdummy, $tdomain, $trest) = split(/\//, $area);
+
+        if ($trole =~ /^cr\//) {
+        # Custom role, defined by a user
+            &custom_roleprivs(\%allroles,$trole,$tdomain,$trest,$spec,$area);
+        } elsif ($trole eq 'gr') {
+        # Role of a member in a group, defined within a course/community
+            &group_roleprivs(\%allgroups,$area,$group_privs,$tend,$tstart);
+            next;
+        } else {
+        # Normal role, defined in roles.tab
+            &standard_roleprivs(\%allroles,$trole,$tdomain,$spec,$trest,$area);
+        }
+
+        my $cid = $tdomain.'_'.$trest;
+        unless ($firstaccchk{$cid}) {
+            if (ref($coursetimerstarts{$cid}) eq 'HASH') {
+                foreach my $item (keys(%{$coursetimerstarts{$cid}})) {
+                    $firstaccenv{'course.'.$cid.'.firstaccess.'.$item} = 
+                        $coursetimerstarts{$cid}{$item}; 
+                }
+            }
+            $firstaccchk{$cid} = 1;
+        }
+        unless ($timerintchk{$cid}) {
+            if (ref($coursetimerintervals{$cid}) eq 'HASH') {
+                foreach my $item (keys(%{$coursetimerintervals{$cid}})) {
+                    $timerintenv{'course.'.$cid.'.timerinterval.'.$item} =
+                       $coursetimerintervals{$cid}{$item};
+                }
+            }
+            $timerintchk{$cid} = 1;
         }
-        my ($author,$adv) = &set_userprivs(\%userroles,\%allroles,\%allgroups);
-        $userroles{'user.adv'}    = $adv;
-	$userroles{'user.author'} = $author;
-        $env{'user.adv'}=$adv;
     }
-    return \%userroles;  
+
+    @userroles{'user.author','user.adv','user.rar'} = &set_userprivs(\%userroles,
+                                                          \%allroles, \%allgroups);
+    $env{'user.adv'} = $userroles{'user.adv'};
+    $env{'user.rar'} = $userroles{'user.rar'};
+
+    return (\%userroles,\%firstaccenv,\%timerintenv);
 }
 
 sub set_arearole {
-    my ($trole,$area,$tstart,$tend,$domain,$username) = @_;
+    my ($trole,$area,$tstart,$tend,$domain,$username,$nolog) = @_;
+    unless ($nolog) {
 # log the associated role with the area
-    &userrolelog($trole,$username,$domain,$area,$tstart,$tend);
+        &userrolelog($trole,$username,$domain,$area,$tstart,$tend);
+    }
     return ('user.role.'.$trole.'.'.$area => $tstart.'.'.$tend);
 }
 
 sub custom_roleprivs {
     my ($allroles,$trole,$tdomain,$trest,$spec,$area) = @_;
     my ($rdummy,$rdomain,$rauthor,$rrole)=split(/\//,$trole);
-    my $homsvr=homeserver($rauthor,$rdomain);
+    my $homsvr = &homeserver($rauthor,$rdomain);
     if (&hostname($homsvr) ne '') {
         my ($rdummy,$roledef)=
             &get('roles',["rolesdef_$rrole"],$rdomain,$rauthor);
@@ -4630,6 +6003,10 @@ sub custom_roleprivs {
                     $$allroles{$spec.'./'.$tdomain.'/'}.=':'.$dompriv;
                 }
                 if (($trest ne '') && (defined($coursepriv))) {
+                    if ($trole =~ m{^cr/$tdomain/$tdomain\Q-domainconfig\E/([^/]+)$}) {
+                        my $rolename = $1;
+                        $coursepriv = &course_adhocrole_privs($rolename,$tdomain,$trest,$coursepriv);
+                    }
                     $$allroles{'cm.'.$area}.=':'.$coursepriv;
                     $$allroles{$spec.'.'.$area}.=':'.$coursepriv;
                 }
@@ -4638,6 +6015,48 @@ sub custom_roleprivs {
     }
 }
 
+sub course_adhocrole_privs {
+    my ($rolename,$cdom,$cnum,$coursepriv) = @_;
+    my %overrides = &get('environment',["internal.adhocpriv.$rolename"],$cdom,$cnum);
+    if ($overrides{"internal.adhocpriv.$rolename"}) {
+        my (%currprivs,%storeprivs);
+        foreach my $item (split(/:/,$coursepriv)) {
+            my ($priv,$restrict) = split(/\&/,$item);
+            $currprivs{$priv} = $restrict;
+        }
+        my (%possadd,%possremove,%full);
+        foreach my $item (split(/\:/,$Apache::lonnet::pr{'cr:c'})) {
+            my ($priv,$restrict)=split(/\&/,$item);
+            $full{$priv} = $restrict;
+        }
+        foreach my $item (split(/,/,$overrides{"internal.adhocpriv.$rolename"})) {
+             next if ($item eq '');
+             my ($rule,$rest) = split(/=/,$item);
+             next unless (($rule eq 'off') || ($rule eq 'on'));
+             foreach my $priv (split(/:/,$rest)) {
+                 if ($priv ne '') {
+                     if ($rule eq 'off') {
+                         $possremove{$priv} = 1;
+                     } else {
+                         $possadd{$priv} = 1;
+                     }
+                 }
+             }
+         }
+         foreach my $priv (sort(keys(%full))) {
+             if (exists($currprivs{$priv})) {
+                 unless (exists($possremove{$priv})) {
+                     $storeprivs{$priv} = $currprivs{$priv};
+                 }
+             } elsif (exists($possadd{$priv})) {
+                 $storeprivs{$priv} = $full{$priv};
+             }
+         }
+         $coursepriv = ':'.join(':',map { $_.'&'.$storeprivs{$_}; } sort(keys(%storeprivs)));
+     }
+     return $coursepriv;
+}
+
 sub group_roleprivs {
     my ($allgroups,$area,$group_privs,$tend,$tstart) = @_;
     my $access = 1;
@@ -4672,6 +6091,7 @@ sub set_userprivs {
     my ($userroles,$allroles,$allgroups,$groups_roles) = @_; 
     my $author=0;
     my $adv=0;
+    my $rar=0;
     my %grouproles = ();
     if (keys(%{$allgroups}) > 0) {
         my @groupkeys; 
@@ -4719,6 +6139,7 @@ sub set_userprivs {
                     $thesepriv{$privilege}.=$restrictions;
                 }
                 if ($thesepriv{'adv'} eq 'F') { $adv=1; }
+                if ($thesepriv{'rar'} eq 'F') { $rar=1; }
             }
         }
         my $thesestr='';
@@ -4727,16 +6148,16 @@ sub set_userprivs {
 	}
         $userroles->{'user.priv.'.$role} = $thesestr;
     }
-    return ($author,$adv);
+    return ($author,$adv,$rar);
 }
 
 sub role_status {
     my ($rolekey,$update,$refresh,$now,$role,$where,$trolecode,$tstatus,$tstart,$tend) = @_;
-    my @pwhere = ();
     if (exists($env{$rolekey}) && $env{$rolekey} ne '') {
-        (undef,undef,$$role,@pwhere)=split(/\./,$rolekey);
+        my ($one,$two) = split(m{\./},$rolekey,2);
+        (undef,undef,$$role) = split(/\./,$one,3);
         unless (!defined($$role) || $$role eq '') {
-            $$where=join('.',@pwhere);
+            $$where = '/'.$two;
             $$trolecode=$$role.'.'.$$where;
             ($$tstart,$$tend)=split(/\./,$env{$rolekey});
             $$tstatus='is';
@@ -4772,9 +6193,10 @@ sub role_status {
                                 push(@rolecodes,$$role);
                                 &standard_roleprivs(\%allroles,$$role,$tdomain,$spec,$trest,$$where);
                             }
-                            my ($author,$adv)= &set_userprivs(\%userroles,\%allroles,\%allgroups,\%groups_roles);
+                            my ($author,$adv,$rar)= &set_userprivs(\%userroles,\%allroles,\%allgroups,
+                                                                   \%groups_roles);
                             &appenv(\%userroles,\@rolecodes);
-                            &log($env{'user.domain'},$env{'user.name'},$env{'user.home'},"Role ".$role);
+                            &log($env{'user.domain'},$env{'user.name'},$env{'user.home'},"Role ".$spec);
                         }
                     }
                     $$tstatus = 'is';
@@ -4850,35 +6272,56 @@ sub delete_env_groupprivs {
 }
 
 sub check_adhoc_privs {
-    my ($cdom,$cnum,$update,$refresh,$now,$checkrole,$caller) = @_;
+    my ($cdom,$cnum,$update,$refresh,$now,$checkrole,$caller,$sec) = @_;
     my $cckey = 'user.role.'.$checkrole.'./'.$cdom.'/'.$cnum;
+    if ($sec) {
+        $cckey .= '/'.$sec;
+    } 
+    my $setprivs;
     if ($env{$cckey}) {
         my ($role,$where,$trolecode,$tstart,$tend,$tremark,$tstatus,$tpstart,$tpend);
         &role_status($cckey,$update,$refresh,$now,\$role,\$where,\$trolecode,\$tstatus,\$tstart,\$tend);
         unless (($tstatus eq 'is') || ($tstatus eq 'will_not')) {
-            &set_adhoc_privileges($cdom,$cnum,$checkrole,$caller);
+            &set_adhoc_privileges($cdom,$cnum,$checkrole,$caller,$sec);
+            $setprivs = 1;
         }
     } else {
-        &set_adhoc_privileges($cdom,$cnum,$checkrole,$caller);
+        &set_adhoc_privileges($cdom,$cnum,$checkrole,$caller,$sec);
+        $setprivs = 1;
     }
+    return $setprivs;
 }
 
 sub set_adhoc_privileges {
-# role can be cc or ca
-    my ($dcdom,$pickedcourse,$role,$caller) = @_;
+# role can be cc, ca, or cr/<dom>/<dom>-domainconfig/role
+    my ($dcdom,$pickedcourse,$role,$caller,$sec) = @_;
     my $area = '/'.$dcdom.'/'.$pickedcourse;
+    if ($sec ne '') {
+        $area .= '/'.$sec;
+    }
     my $spec = $role.'.'.$area;
     my %userroles = &set_arearole($role,$area,'','',$env{'user.domain'},
-                                  $env{'user.name'});
-    my %ccrole = ();
-    &standard_roleprivs(\%ccrole,$role,$dcdom,$spec,$pickedcourse,$area);
-    my ($author,$adv)= &set_userprivs(\%userroles,\%ccrole);
+                                  $env{'user.name'},1);
+    my %rolehash = ();
+    if ($role =~ m{^\Qcr/$dcdom/$dcdom\E\-domainconfig/(\w+)$}) {
+        my $rolename = $1;
+        &custom_roleprivs(\%rolehash,$role,$dcdom,$pickedcourse,$spec,$area);
+        my %domdef = &get_domain_defaults($dcdom);
+        if (ref($domdef{'adhocroles'}) eq 'HASH') {
+            if (ref($domdef{'adhocroles'}{$rolename}) eq 'HASH') {
+                &appenv({'request.role.desc' => $domdef{'adhocroles'}{$rolename}{'desc'},});
+            }
+        }
+    } else {
+        &standard_roleprivs(\%rolehash,$role,$dcdom,$spec,$pickedcourse,$area);
+    }
+    my ($author,$adv,$rar)= &set_userprivs(\%userroles,\%rolehash);
     &appenv(\%userroles,[$role,'cm']);
-    &log($env{'user.domain'},$env{'user.name'},$env{'user.home'},"Role ".$role);
+    &log($env{'user.domain'},$env{'user.name'},$env{'user.home'},"Role ".$spec);
     unless ($caller eq 'constructaccess' && $env{'request.course.id'}) {
         &appenv( {'request.role'        => $spec,
                   'request.role.domain' => $dcdom,
-                  'request.course.sec'  => ''
+                  'request.course.sec'  => $sec,
                  }
                );
         my $tadv=0;
@@ -4932,23 +6375,49 @@ sub del {
 
 # -------------------------------------------------------------- dump interface
 
+sub unserialize {
+    my ($rep, $escapedkeys) = @_;
+
+    return {} if $rep =~ /^error/;
+
+    my %returnhash=();
+	foreach my $item (split(/\&/,$rep)) {
+	    my ($key, $value) = split(/=/, $item, 2);
+	    $key = unescape($key) unless $escapedkeys;
+	    next if $key =~ /^error: 2 /;
+	    $returnhash{$key} = &thaw_unescape($value);
+	}
+    #return %returnhash;
+    return \%returnhash;
+}        
+
+# see Lond::dump_with_regexp
+# if $escapedkeys hash keys won't get unescaped.
 sub dump {
-    my ($namespace,$udomain,$uname,$regexp,$range,$extra)=@_;
+    my ($namespace,$udomain,$uname,$regexp,$range,$escapedkeys)=@_;
     if (!$udomain) { $udomain=$env{'user.domain'}; }
     if (!$uname) { $uname=$env{'user.name'}; }
     my $uhome=&homeserver($uname,$udomain);
+
     if ($regexp) {
-	$regexp=&escape($regexp);
+        $regexp=&escape($regexp);
     } else {
-	$regexp='.';
+        $regexp='.';
     }
-    my $rep=&reply("dump:$udomain:$uname:$namespace:$regexp:$range:$extra",$uhome);
+    if (grep { $_ eq $uhome } current_machine_ids()) {
+        # user is hosted on this machine
+        my $reply = LONCAPA::Lond::dump_with_regexp(join(":", ($udomain,
+                    $uname, $namespace, $regexp, $range)), $perlvar{'lonVersion'});
+        return %{unserialize($reply, $escapedkeys)};
+    }
+    my $rep=&reply("dump:$udomain:$uname:$namespace:$regexp:$range",$uhome);
     my @pairs=split(/\&/,$rep);
     my %returnhash=();
     if (!($rep =~ /^error/ )) {
 	foreach my $item (@pairs) {
 	    my ($key,$value)=split(/=/,$item,2);
-	    $key = &unescape($key);
+        $key = unescape($key) unless $escapedkeys;
+        #$key = &unescape($key);
 	    next if ($key =~ /^error: 2 /);
 	    $returnhash{$key}=&thaw_unescape($value);
 	}
@@ -4961,23 +6430,9 @@ sub dump {
 
 sub dumpstore {
    my ($namespace,$udomain,$uname,$regexp,$range)=@_;
-   if (!$udomain) { $udomain=$env{'user.domain'}; }
-   if (!$uname) { $uname=$env{'user.name'}; }
-   my $uhome=&homeserver($uname,$udomain);
-   if ($regexp) {
-       $regexp=&escape($regexp);
-   } else {
-       $regexp='.';
-   }
-   my $rep=&reply("dump:$udomain:$uname:$namespace:$regexp:$range",$uhome);
-   my @pairs=split(/\&/,$rep);
-   my %returnhash=();
-   foreach my $item (@pairs) {
-       my ($key,$value)=split(/=/,$item,2);
-       next if ($key =~ /^error: 2 /);
-       $returnhash{$key}=&thaw_unescape($value);
-   }
-   return %returnhash;
+   # same as dump but keys must be escaped. They may contain colon separated
+   # lists of values that may themself contain colons (e.g. symbs).
+   return &dump($namespace, $udomain, $uname, $regexp, $range, 1);
 }
 
 # -------------------------------------------------------------- keys interface
@@ -5003,12 +6458,20 @@ sub currentdump {
    $sdom     = $env{'user.domain'}       if (! defined($sdom));
    $sname    = $env{'user.name'}         if (! defined($sname));
    my $uhome = &homeserver($sname,$sdom);
-   my $rep=reply('currentdump:'.$sdom.':'.$sname.':'.$courseid,$uhome);
+   my $rep;
+
+   if (grep { $_ eq $uhome } current_machine_ids()) {
+       $rep = LONCAPA::Lond::dump_profile_database(join(":", ($sdom, $sname, 
+                   $courseid)));
+   } else {
+       $rep = reply('currentdump:'.$sdom.':'.$sname.':'.$courseid,$uhome);
+   }
+
    return if ($rep =~ /^(error:|no_such_host)/);
    #
    my %returnhash=();
    #
-   if ($rep eq "unknown_cmd") { 
+   if ($rep eq 'unknown_cmd') {
        # an old lond will not know currentdump
        # Do a dump and make it look like a currentdump
        my @tmp = &dumpstore($courseid,$sdom,$sname,'.');
@@ -5124,7 +6587,7 @@ sub newput {
 # ---------------------------------------------------------  putstore interface
 
 sub putstore {
-   my ($namespace,$symb,$version,$storehash,$udomain,$uname)=@_;
+   my ($namespace,$symb,$version,$storehash,$udomain,$uname,$tolog)=@_;
    if (!$udomain) { $udomain=$env{'user.domain'}; }
    if (!$uname) { $uname=$env{'user.name'}; }
    my $uhome=&homeserver($uname,$udomain);
@@ -5138,6 +6601,17 @@ sub putstore {
    my $reply =
        &reply("putstore:$udomain:$uname:$namespace:$esc_symb:$esc_v:$items",
 	      $uhome);
+   if (($tolog) && ($reply eq 'ok')) {
+       my $namevalue='';
+       foreach my $key (keys(%{$storehash})) {
+           $namevalue.=&escape($key).'='.&freeze_escape($storehash->{$key}).'&';
+       }
+       $namevalue .= 'ip='.&escape($ENV{'REMOTE_ADDR'}).
+                     '&host='.&escape($perlvar{'lonHostID'}).
+                     '&version='.$esc_v.
+                     '&by='.&escape($env{'user.name'}.':'.$env{'user.domain'});
+       &Apache::lonnet::courselog($symb.':'.$uname.':'.$udomain.':PUTSTORE:'.$namevalue);
+   }
    if ($reply eq 'unknown_cmd') {
        # gfall back to way things use to be done
        return &old_putstore($namespace,$symb,$version,$storehash,$udomain,
@@ -5231,9 +6705,11 @@ sub tmpget {
     if (!defined($server)) { $server = $perlvar{'lonHostID'}; }
     my $rep=&reply("tmpget:$token",$server);
     my %returnhash;
+    if ($rep =~ /^(con_lost|error|no_such_host)/i) {
+        return %returnhash;
+    }
     foreach my $item (split(/\&/,$rep)) {
 	my ($key,$value)=split(/=/,$item);
-        next if ($key =~ /^error: 2 /);
 	$returnhash{&unescape($key)}=&thaw_unescape($value);
     }
     return %returnhash;
@@ -5246,12 +6722,111 @@ sub tmpdel {
     return &reply("tmpdel:$token",$server);
 }
 
+# ------------------------------------------------------------ get_timebased_id 
+
+sub get_timebased_id {
+    my ($prefix,$keyid,$namespace,$cdom,$cnum,$idtype,$who,$locktries,
+        $maxtries) = @_;
+    my ($newid,$error,$dellock);
+    unless (($prefix =~ /^\w+$/) && ($keyid =~ /^\w+$/) && ($namespace ne '')) {  
+        return ('','ok','invalid call to get suffix');
+    }
+
+# set defaults for any optional args for which values were not supplied
+    if ($who eq '') {
+        $who = $env{'user.name'}.':'.$env{'user.domain'};
+    }
+    if (!$locktries) {
+        $locktries = 3;
+    }
+    if (!$maxtries) {
+        $maxtries = 10;
+    }
+    
+    if (($cdom eq '') || ($cnum eq '')) {
+        if ($env{'request.course.id'}) {
+            $cdom = $env{'course.'.$env{'request.course.id'}.'.domain'};
+            $cnum = $env{'course.'.$env{'request.course.id'}.'.num'};
+        }
+        if (($cdom eq '') || ($cnum eq '')) {
+            return ('','ok','call to get suffix not in course context');
+        }
+    }
+
+# construct locking item
+    my $lockhash = {
+                      $prefix."\0".'locked_'.$keyid => $who,
+                   };
+    my $tries = 0;
+
+# attempt to get lock on nohist_$namespace file
+    my $gotlock = &Apache::lonnet::newput('nohist_'.$namespace,$lockhash,$cdom,$cnum);
+    while (($gotlock ne 'ok') && $tries <$locktries) {
+        $tries ++;
+        sleep 1;
+        $gotlock = &Apache::lonnet::newput('nohist_'.$namespace,$lockhash,$cdom,$cnum);
+    }
+
+# attempt to get unique identifier, based on current timestamp
+    if ($gotlock eq 'ok') {
+        my %inuse = &Apache::lonnet::dump('nohist_'.$namespace,$cdom,$cnum,$prefix);
+        my $id = time;
+        $newid = $id;
+        if ($idtype eq 'addcode') {
+            $newid .= &sixnum_code();
+        }
+        my $idtries = 0;
+        while (exists($inuse{$prefix."\0".$newid}) && $idtries < $maxtries) {
+            if ($idtype eq 'concat') {
+                $newid = $id.$idtries;
+            } elsif ($idtype eq 'addcode') {
+                $newid = $newid.&sixnum_code();
+            } else {
+                $newid ++;
+            }
+            $idtries ++;
+        }
+        if (!exists($inuse{$prefix."\0".$newid})) {
+            my %new_item =  (
+                              $prefix."\0".$newid => $who,
+                            );
+            my $putresult = &Apache::lonnet::put('nohist_'.$namespace,\%new_item,
+                                                 $cdom,$cnum);
+            if ($putresult ne 'ok') {
+                undef($newid);
+                $error = 'error saving new item: '.$putresult;
+            }
+        } else {
+             undef($newid);
+             $error = ('error: no unique suffix available for the new item ');
+        }
+#  remove lock
+        my @del_lock = ($prefix."\0".'locked_'.$keyid);
+        $dellock = &Apache::lonnet::del('nohist_'.$namespace,\@del_lock,$cdom,$cnum);
+    } else {
+        $error = "error: could not obtain lockfile\n";
+        $dellock = 'ok';
+        if (($prefix eq 'paste') && ($namespace eq 'courseeditor') && ($keyid eq 'num')) {
+            $dellock = 'nolock';
+        }
+    }
+    return ($newid,$dellock,$error);
+}
+
+sub sixnum_code {
+    my $code;
+    for (0..6) {
+        $code .= int( rand(9) );
+    }
+    return $code;
+}
+
 # -------------------------------------------------- portfolio access checking
 
 sub portfolio_access {
-    my ($requrl) = @_;
+    my ($requrl,$clientip) = @_;
     my (undef,$udom,$unum,$file_name,$group) = &parse_portfolio_url($requrl);
-    my $result = &get_portfolio_access($udom,$unum,$file_name,$group);
+    my $result = &get_portfolio_access($udom,$unum,$file_name,$group,$clientip);
     if ($result) {
         my %setters;
         if ($env{'user.name'} eq 'public' && $env{'user.domain'} eq 'public') {
@@ -5277,7 +6852,7 @@ sub portfolio_access {
 }
 
 sub get_portfolio_access {
-    my ($udom,$unum,$file_name,$group,$access_hash) = @_;
+    my ($udom,$unum,$file_name,$group,$clientip,$access_hash) = @_;
 
     if (!ref($access_hash)) {
 	my $current_perms = &get_portfile_permissions($udom,$unum);
@@ -5286,7 +6861,7 @@ sub get_portfolio_access {
 	$access_hash = $access_controls{$file_name};
     }
 
-    my ($public,$guest,@domains,@users,@courses,@groups);
+    my ($public,$guest,@domains,@users,@courses,@groups,@ips);
     my $now = time;
     if (ref($access_hash) eq 'HASH') {
         foreach my $key (keys(%{$access_hash})) {
@@ -5310,10 +6885,25 @@ sub get_portfolio_access {
                 push(@courses,$key);
             } elsif ($scope eq 'group') {
                 push(@groups,$key);
+            } elsif ($scope eq 'ip') {
+                push(@ips,$key);
             }
         }
         if ($public) {
             return 'ok';
+        } elsif (@ips > 0) {
+            my $allowed;
+            foreach my $ipkey (@ips) {
+                if (ref($access_hash->{$ipkey}{'ip'}) eq 'ARRAY') {
+                    if (&Apache::loncommon::check_ip_acc(join(',',@{$access_hash->{$ipkey}{'ip'}}),$clientip)) {
+                        $allowed = 1;
+                        last; 
+                    }
+                }
+            }
+            if ($allowed) {
+                return 'ok';
+            }
         }
         if ($env{'user.name'} eq 'public' && $env{'user.domain'} eq 'public') {
             if ($guest) {
@@ -5500,17 +7090,24 @@ sub usertools_access {
                       official   => 1,
                       unofficial => 1,
                       community  => 1,
+                      textbook   => 1,
+                      placement  => 1,
+                 );
+    } elsif ($context eq 'requestauthor') {
+        %tools = (
+                      requestauthor => 1,
                  );
     } else {
         %tools = (
                       aboutme   => 1,
                       blog      => 1,
+                      webdav    => 1,
                       portfolio => 1,
                  );
     }
     return if (!defined($tools{$tool}));
 
-    if ((!defined($udom)) || (!defined($uname))) {
+    if (($udom eq '') || ($uname eq '')) {
         $udom = $env{'user.domain'};
         $uname = $env{'user.name'};
     }
@@ -5519,25 +7116,32 @@ sub usertools_access {
         if ($action ne 'reload') {
             if ($context eq 'requestcourses') {
                 return $env{'environment.canrequest.'.$tool};
+            } elsif ($context eq 'requestauthor') {
+                return $env{'environment.canrequest.author'};
             } else {
                 return $env{'environment.availabletools.'.$tool};
             }
         }
     }
 
-    my ($toolstatus,$inststatus);
+    my ($toolstatus,$inststatus,$envkey);
+    if ($context eq 'requestauthor') {
+        $envkey = $context; 
+    } else {
+        $envkey = $context.'.'.$tool;
+    }
 
     if (($udom eq $env{'user.domain'}) && ($uname eq $env{'user.name'}) &&
          ($action ne 'reload')) {
-        $toolstatus = $env{'environment.'.$context.'.'.$tool};
+        $toolstatus = $env{'environment.'.$envkey};
         $inststatus = $env{'environment.inststatus'};
     } else {
         if (ref($userenvref) eq 'HASH') {
-            $toolstatus = $userenvref->{$context.'.'.$tool};
+            $toolstatus = $userenvref->{$envkey};
             $inststatus = $userenvref->{'inststatus'};
         } else {
-            my %userenv = &userenvironment($udom,$uname,$context.'.'.$tool,'inststatus');
-            $toolstatus = $userenv{$context.'.'.$tool};
+            my %userenv = &userenvironment($udom,$uname,$envkey,'inststatus');
+            $toolstatus = $userenv{$envkey};
             $inststatus = $userenv{'inststatus'};
         }
     }
@@ -5603,7 +7207,7 @@ sub usertools_access {
             }
         }
     } else {
-        if ($context eq 'tools') {
+        if (($context eq 'tools') && ($tool ne 'webdav')) {
             $access = 1;
         } else {
             $access = 0;
@@ -5784,7 +7388,7 @@ sub customaccess {
 # ------------------------------------------------- Check for a user privilege
 
 sub allowed {
-    my ($priv,$uri,$symb,$role)=@_;
+    my ($priv,$uri,$symb,$role,$clientip,$noblockcheck)=@_;
     my $ver_orguri=$uri;
     $uri=&deversion($uri);
     my $orguri=$uri;
@@ -5801,7 +7405,7 @@ sub allowed {
 
     if (defined($env{'allowed.'.$priv})) { return $env{'allowed.'.$priv}; }
 # Free bre access to adm and meta resources
-    if (((($uri=~/^adm\//) && ($uri !~ m{/(?:smppg|bulletinboard)$})) 
+    if (((($uri=~/^adm\//) && ($uri !~ m{/(?:smppg|bulletinboard|ext\.tool)$})) 
 	 || (($uri=~/\.meta$/) && ($uri!~m|^uploaded/|) )) 
 	&& ($priv eq 'bre')) {
 	return 'F';
@@ -5979,7 +7583,17 @@ sub allowed {
         if ($match) {
             if ($env{'user.priv.'.$env{'request.role'}.'./'}
                   =~/\Q$priv\E\&([^\:]*)/) {
-                $thisallowed.=$1;
+                my $value = $1;
+                if ($noblockcheck) {
+                    $thisallowed.=$value;
+                } else {
+                    my @blockers = &has_comm_blocking($priv,$symb,$uri);
+                    if (@blockers > 0) {
+                        $thisallowed = 'B';
+                    } else {
+                        $thisallowed.=$value;
+                    }
+                }
             }
         } else {
             my $refuri = $env{'httpref.'.$orguri} || $env{'httpref.'.$ver_orguri};
@@ -5990,7 +7604,16 @@ sub allowed {
                     $refuri=&declutter($refuri);
                     my ($match) = &is_on_map($refuri);
                     if ($match) {
-                        $thisallowed='F';
+                        if ($noblockcheck) {
+                            $thisallowed='F';
+                        } else {
+                            my @blockers = &has_comm_blocking($priv,$symb,$refuri);
+                            if (@blockers > 0) {
+                                $thisallowed = 'B';
+                            } else {
+                                $thisallowed='F';
+                            }
+                        }
                     }
                 }
             }
@@ -6001,9 +7624,9 @@ sub allowed {
 	&& $thisallowed ne 'F' 
 	&& $thisallowed ne '2'
 	&& &is_portfolio_url($uri)) {
-	$thisallowed = &portfolio_access($uri);
+	$thisallowed = &portfolio_access($uri,$clientip);
     }
-    
+
 # Full access at system, domain or course-wide level? Exit.
     if ($thisallowed=~/F/) {
 	return 'F';
@@ -6042,7 +7665,21 @@ sub allowed {
            $statecond=$cond;
            if ($env{'user.priv.'.$env{'request.role'}.'./'.$courseprivid}
                =~/\Q$priv\E\&([^\:]*)/) {
-               $thisallowed.=$1;
+               my $value = $1;
+               if ($priv eq 'bre') {
+                   if ($noblockcheck) {
+                       $thisallowed.=$value;
+                   } else {
+                       my @blockers = &has_comm_blocking($priv,$symb,$uri);
+                       if (@blockers > 0) {
+                           $thisallowed = 'B';
+                       } else {
+                           $thisallowed.=$value;
+                       }
+                   }
+               } else {
+                   $thisallowed.=$value;
+               }
                $checkreferer=0;
            }
        }
@@ -6070,7 +7707,21 @@ sub allowed {
               my $refstatecond=$cond;
               if ($env{'user.priv.'.$env{'request.role'}.'./'.$courseprivid}
                   =~/\Q$priv\E\&([^\:]*)/) {
-                  $thisallowed.=$1;
+                  my $value = $1;
+                  if ($priv eq 'bre') {
+                      if ($noblockcheck) {
+                          $thisallowed.=$value;
+                      } else {
+                          my @blockers = &has_comm_blocking($priv,$symb,$refuri);
+                          if (@blockers > 0) {
+                              $thisallowed = 'B';
+                          } else {
+                              $thisallowed.=$value;
+                          }
+                      }
+                  } else {
+                      $thisallowed.=$value;
+                  }
                   $uri=$refuri;
                   $statecond=$refstatecond;
               }
@@ -6174,7 +7825,7 @@ sub allowed {
        my $unamedom=$env{'user.name'}.':'.$env{'user.domain'};
        if ($env{'course.'.$env{'request.course.id'}.'.'.$priv.'.roles.denied'}
 	   =~/\Q$rolecode\E/) {
-	   if (($priv ne 'pch') && ($priv ne 'plc')) { 
+	   if (($priv ne 'pch') && ($priv ne 'plc') && ($priv ne 'pac')) {
 	       &logthis($env{'user.domain'}.':'.$env{'user.name'}.':'.$env{'user.home'}.':'.
 			'Denied by role: '.$priv.' for '.$uri.' as '.$rolecode.' in '.
 			$env{'request.course.id'});
@@ -6184,7 +7835,7 @@ sub allowed {
 
        if ($env{'course.'.$env{'request.course.id'}.'.'.$priv.'.users.denied'}
 	   =~/\Q$unamedom\E/) {
-	   if (($priv ne 'pch') && ($priv ne 'plc')) { 
+	   if (($priv ne 'pch') && ($priv ne 'plc') && ($priv ne 'pac')) {
 	       &logthis($env{'user.domain'}.':'.$env{'user.name'}.':'.$env{'user.home'}.
 			'Denied by user: '.$priv.' for '.$uri.' as '.$unamedom.' in '.
 			$env{'request.course.id'});
@@ -6229,8 +7880,300 @@ sub allowed {
     }
    return 'F';
 }
+
+# ------------------------------------------- Check construction space access
+
+sub constructaccess {
+    my ($url,$setpriv)=@_;
+
+# We do not allow editing of previous versions of files
+    if ($url=~/\.(\d+)\.(\w+)$/) { return ''; }
+
+# Get username and domain from URL
+    my ($ownername,$ownerdomain,$ownerhome);
+
+    ($ownerdomain,$ownername) =
+        ($url=~ m{^(?:\Q$perlvar{'lonDocRoot'}\E|)(?:/daxepage|/daxeopen)?/priv/($match_domain)/($match_username)(?:/|$)});
+
+# The URL does not really point to any authorspace, forget it
+    unless (($ownername) && ($ownerdomain)) { return ''; }
+
+# Now we need to see if the user has access to the authorspace of
+# $ownername at $ownerdomain
+
+    if (($ownername eq $env{'user.name'}) && ($ownerdomain eq $env{'user.domain'})) {
+# Real author for this?
+       $ownerhome = $env{'user.home'};
+       if (exists($env{'user.priv.au./'.$ownerdomain.'/./'})) {
+          return ($ownername,$ownerdomain,$ownerhome);
+       }
+    } else {
+# Co-author for this?
+        if (exists($env{'user.priv.ca./'.$ownerdomain.'/'.$ownername.'./'}) ||
+            exists($env{'user.priv.aa./'.$ownerdomain.'/'.$ownername.'./'}) ) {
+            $ownerhome = &homeserver($ownername,$ownerdomain);
+            return ($ownername,$ownerdomain,$ownerhome);
+        }
+        if ($env{'request.course.id'}) {
+            if (($ownername eq $env{'course.'.$env{'request.course.id'}.'.num'}) &&
+                ($ownerdomain eq $env{'course.'.$env{'request.course.id'}.'.domain'})) {
+                if (&allowed('mdc',$env{'request.course.id'})) {
+                    $ownerhome = $env{'course.'.$env{'request.course.id'}.'.home'};
+                    return ($ownername,$ownerdomain,$ownerhome);
+                }
+            }
+        }
+    }
+
+# We don't have any access right now. If we are not possibly going to do anything about this,
+# we might as well leave
+   unless ($setpriv) { return ''; }
+
+# Backdoor access?
+    my $allowed=&allowed('eco',$ownerdomain);
+# Nope
+    unless ($allowed) { return ''; }
+# Looks like we may have access, but could be locked by the owner of the construction space
+    if ($allowed eq 'U') {
+        my %blocked=&get('environment',['domcoord.author'],
+                         $ownerdomain,$ownername);
+# Is blocked by owner
+        if ($blocked{'domcoord.author'} eq 'blocked') { return ''; }
+    }
+    if (($allowed eq 'F') || ($allowed eq 'U')) {
+# Grant temporary access
+        my $then=$env{'user.login.time'};
+        my $update=$env{'user.update.time'};
+        if (!$update) { $update = $then; }
+        my $refresh=$env{'user.refresh.time'};
+        if (!$refresh) { $refresh = $update; }
+        my $now = time;
+        &check_adhoc_privs($ownerdomain,$ownername,$update,$refresh,
+                           $now,'ca','constructaccess');
+        $ownerhome = &homeserver($ownername,$ownerdomain);
+        return($ownername,$ownerdomain,$ownerhome);
+    }
+# No business here
+    return '';
+}
+
+# ----------------------------------------------------------- Content Blocking
+
+{
+# Caches for faster Course Contents display where content blocking
+# is in operation (i.e., interval param set) for timed quiz.
+#
+# User for whom data are being temporarily cached.
+my $cacheduser='';
+# Cached blockers for this user (a hash of blocking items). 
+my %cachedblockers=();
+# When the data were last cached.
+my $cachedlast='';
+
+sub load_all_blockers {
+    my ($uname,$udom,$blocks)=@_;
+    if (($uname ne '') && ($udom ne '')) { 
+        if (($cacheduser eq $uname.':'.$udom) &&
+            (abs($cachedlast-time)<5)) {
+            return;
+        }
+    }
+    $cachedlast=time;
+    $cacheduser=$uname.':'.$udom;
+    %cachedblockers = &get_commblock_resources($blocks);
+}
+
+sub get_comm_blocks {
+    my ($cdom,$cnum) = @_;
+    if ($cdom eq '' || $cnum eq '') {
+        return unless ($env{'request.course.id'});
+        $cnum = $env{'course.'.$env{'request.course.id'}.'.num'};
+        $cdom = $env{'course.'.$env{'request.course.id'}.'.domain'};
+    }
+    my %commblocks;
+    my $hashid=$cdom.'_'.$cnum;
+    my ($blocksref,$cached)=&is_cached_new('comm_block',$hashid);
+    if ((defined($cached)) && (ref($blocksref) eq 'HASH')) {
+        %commblocks = %{$blocksref};
+    } else {
+        %commblocks = &Apache::lonnet::dump('comm_block',$cdom,$cnum);
+        my $cachetime = 600;
+        &do_cache_new('comm_block',$hashid,\%commblocks,$cachetime);
+    }
+    return %commblocks;
+}
+
+sub get_commblock_resources {
+    my ($blocks) = @_;
+    my %blockers = ();
+    return %blockers unless ($env{'request.course.id'});
+    return %blockers if ($env{'user.priv.'.$env{'request.role'}} =~/evb\&([^\:]*)/);
+    my %commblocks;
+    if (ref($blocks) eq 'HASH') {
+        %commblocks = %{$blocks};
+    } else {
+        %commblocks = &get_comm_blocks();
+    }
+    return %blockers unless (keys(%commblocks) > 0); 
+    my $navmap = Apache::lonnavmaps::navmap->new();
+    return %blockers unless (ref($navmap));
+    my $now = time;
+    foreach my $block (keys(%commblocks)) {
+        if ($block =~ /^(\d+)____(\d+)$/) {
+            my ($start,$end) = ($1,$2);
+            if ($start <= $now && $end >= $now) {
+                if (ref($commblocks{$block}{'blocks'}) eq 'HASH') {
+                    if (ref($commblocks{$block}{'blocks'}{'docs'}) eq 'HASH') {
+                        if (ref($commblocks{$block}{'blocks'}{'docs'}{'maps'}) eq 'HASH') {
+                            if (keys(%{$commblocks{$block}{'blocks'}{'docs'}{'maps'}})) {
+                                $blockers{$block}{maps} = $commblocks{$block}{'blocks'}{'docs'}{'maps'}; 
+                            }
+                        }
+                        if (ref($commblocks{$block}{'blocks'}{'docs'}{'resources'}) eq 'HASH') {
+                            if (keys(%{$commblocks{$block}{'blocks'}{'docs'}{'resources'}})) {
+                                $blockers{$block}{'resources'} = $commblocks{$block}{'blocks'}{'docs'}{'resources'};
+                            }
+                        }
+                    }
+                }
+            }
+        } elsif ($block =~ /^firstaccess____(.+)$/) {
+            my $item = $1;
+            my @to_test;
+            if (ref($commblocks{$block}{'blocks'}) eq 'HASH') {
+                if (ref($commblocks{$block}{'blocks'}{'docs'}) eq 'HASH') {
+                    my @interval;
+                    my $type = 'map';
+                    if ($item eq 'course') {
+                        $type = 'course';
+                        @interval=&EXT("resource.0.interval");
+                    } else {
+                        if ($item =~ /___\d+___/) {
+                            $type = 'resource';
+                            @interval=&EXT("resource.0.interval",$item);
+                            if (ref($navmap)) {                        
+                                my $res = $navmap->getBySymb($item); 
+                                push(@to_test,$res);
+                            }
+                        } else {
+                            my $mapsymb = &symbread($item,1);
+                            if ($mapsymb) {
+                                if (ref($navmap)) {
+                                    my $mapres = $navmap->getBySymb($mapsymb);
+                                    @to_test = $mapres->retrieveResources($mapres,undef,0,0,0,1);
+                                    foreach my $res (@to_test) {
+                                        my $symb = $res->symb();
+                                        next if ($symb eq $mapsymb);
+                                        if ($symb ne '') {
+                                            @interval=&EXT("resource.0.interval",$symb);
+                                            if ($interval[1] eq 'map') {
+                                                last;
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    if ($interval[0] =~ /^(\d+)/) {
+                        my $timelimit = $1; 
+                        my $first_access;
+                        if ($type eq 'resource') {
+                            $first_access=&get_first_access($interval[1],$item);
+                        } elsif ($type eq 'map') {
+                            $first_access=&get_first_access($interval[1],undef,$item);
+                        } else {
+                            $first_access=&get_first_access($interval[1]);
+                        }
+                        if ($first_access) {
+                            my $timesup = $first_access+$timelimit;
+                            if ($timesup > $now) {
+                                my $activeblock;
+                                foreach my $res (@to_test) {
+                                    if ($res->answerable()) {
+                                        $activeblock = 1;
+                                        last;
+                                    }
+                                }
+                                if ($activeblock) {
+                                    if (ref($commblocks{$block}{'blocks'}{'docs'}{'maps'}) eq 'HASH') {
+                                         if (keys(%{$commblocks{$block}{'blocks'}{'docs'}{'maps'}})) {
+                                             $blockers{$block}{'maps'} = $commblocks{$block}{'blocks'}{'docs'}{'maps'};
+                                         }
+                                    }
+                                    if (ref($commblocks{$block}{'blocks'}{'docs'}{'resources'}) eq 'HASH') {
+                                        if (keys(%{$commblocks{$block}{'blocks'}{'docs'}{'resources'}})) {
+                                            $blockers{$block}{'resources'} = $commblocks{$block}{'blocks'}{'docs'}{'resources'};
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    return %blockers;
+}
+
+sub has_comm_blocking {
+    my ($priv,$symb,$uri,$blocks) = @_;
+    my @blockers;
+    return unless ($env{'request.course.id'});
+    return unless ($priv eq 'bre');
+    return if ($env{'user.priv.'.$env{'request.role'}} =~/evb\&([^\:]*)/);
+    return if ($env{'request.state'} eq 'construct');
+    &load_all_blockers($env{'user.name'},$env{'user.domain'},$blocks);
+    return unless (keys(%cachedblockers) > 0);
+    my (%possibles,@symbs);
+    if (!$symb) {
+        $symb = &symbread($uri,1,1,1,\%possibles);
+    }
+    if ($symb) {
+        @symbs = ($symb);
+    } elsif (keys(%possibles)) { 
+        @symbs = keys(%possibles);
+    }
+    my $noblock;
+    foreach my $symb (@symbs) {
+        last if ($noblock);
+        my ($map,$resid,$resurl)=&decode_symb($symb);
+        foreach my $block (keys(%cachedblockers)) {
+            if ($block =~ /^firstaccess____(.+)$/) {
+                my $item = $1;
+                if (($item eq $map) || ($item eq $symb)) {
+                    $noblock = 1;
+                    last;
+                }
+            }
+            if (ref($cachedblockers{$block}) eq 'HASH') {
+                if (ref($cachedblockers{$block}{'resources'}) eq 'HASH') {
+                    if ($cachedblockers{$block}{'resources'}{$symb}) {
+                        unless (grep(/^\Q$block\E$/,@blockers)) {
+                            push(@blockers,$block);
+                        }
+                    }
+                }
+            }
+            if (ref($cachedblockers{$block}{'maps'}) eq 'HASH') {
+                if ($cachedblockers{$block}{'maps'}{$map}) {
+                    unless (grep(/^\Q$block\E$/,@blockers)) {
+                        push(@blockers,$block);
+                    }
+                }
+            }
+        }
+    }
+    return if ($noblock);
+    return @blockers;
+}
+}
+
+# -------------------------------- Deversion and split uri into path an filename   
+
 #
-#   Removes the versino from a URI and
+#   Removes the version from a URI and
 #   splits it in to its filename and path to the filename.
 #   Seems like File::Basename could have done this more clearly.
 #   Parameters:
@@ -6298,7 +8241,7 @@ sub get_symb_from_alias {
 
 sub definerole {
   if (allowed('mcr','/')) {
-    my ($rolename,$sysrole,$domrole,$courole)=@_;
+    my ($rolename,$sysrole,$domrole,$courole,$uname,$udom)=@_;
     foreach my $role (split(':',$sysrole)) {
 	my ($crole,$cqual)=split(/\&/,$role);
         if ($pr{'cr:s'}!~/\Q$crole\E/) { return "refused:s:$crole"; }
@@ -6326,11 +8269,19 @@ sub definerole {
             }
         }
     }
+    my $uhome;
+    if (($uname ne '') && ($udom ne '')) {
+        $uhome = &homeserver($uname,$udom);
+        return $uhome if ($uhome eq 'no_host');
+    } else {
+        $uname = $env{'user.name'};
+        $udom = $env{'user.domain'};
+        $uhome = $env{'user.home'};
+    }
     my $command="encrypt:rolesput:$env{'user.domain'}:$env{'user.name'}:".
-                "$env{'user.domain'}:$env{'user.name'}:".
-	        "rolesdef_$rolename=".
+                "$udom:$uname:rolesdef_$rolename=".
                 escape($sysrole.'_'.$domrole.'_'.$courole);
-    return reply($command,$env{'user.home'});
+    return reply($command,$uhome);
   } else {
     return 'refused';
   }
@@ -6339,19 +8290,23 @@ sub definerole {
 # ---------------- Make a metadata query against the network of library servers
 
 sub metadata_query {
-    my ($query,$custom,$customshow,$server_array)=@_;
+    my ($query,$custom,$customshow,$server_array,$domains_hash)=@_;
     my %rhash;
     my %libserv = &all_library();
     my @server_list = (defined($server_array) ? @$server_array
                                               : keys(%libserv) );
     for my $server (@server_list) {
+        my $domains = ''; 
+        if (ref($domains_hash) eq 'HASH') {
+            $domains = $domains_hash->{$server}; 
+        }
 	unless ($custom or $customshow) {
-	    my $reply=&reply("querysend:".&escape($query),$server);
+	    my $reply=&reply("querysend:".&escape($query).':::'.&escape($domains),$server);
 	    $rhash{$server}=$reply;
 	}
 	else {
 	    my $reply=&reply("querysend:".&escape($query).':'.
-			     &escape($custom).':'.&escape($customshow),
+			     &escape($custom).':'.&escape($customshow).':'.&escape($domains),
 			     $server);
 	    $rhash{$server}=$reply;
 	}
@@ -6408,10 +8363,12 @@ sub update_allusers_table {
 
 sub fetch_enrollment_query {
     my ($context,$affiliatesref,$replyref,$dom,$cnum) = @_;
-    my $homeserver;
+    my ($homeserver,$sleep,$loopmax);
     my $maxtries = 1;
     if ($context eq 'automated') {
         $homeserver = $perlvar{'lonHostID'};
+        $sleep = 2;
+        $loopmax = 100;
         $maxtries = 10; # will wait for up to 2000s for retrieval of classlist data before timeout
     } else {
         $homeserver = &homeserver($cnum,$dom);
@@ -6429,17 +8386,17 @@ sub fetch_enrollment_query {
         &logthis('fetch_enrollment_query: invalid queryid: '.$queryid.' for host: '.$host.' and homeserver: '.$homeserver.' context: '.$context.' '.$cnum); 
         return 'error: '.$queryid;
     }
-    my $reply = &get_query_reply($queryid);
+    my $reply = &get_query_reply($queryid,$sleep,$loopmax);
     my $tries = 1;
     while (($reply=~/^timeout/) && ($tries < $maxtries)) {
-        $reply = &get_query_reply($queryid);
+        $reply = &get_query_reply($queryid,$sleep,$loopmax);
         $tries ++;
     }
     if ( ($reply =~/^timeout/) || ($reply =~/^error/) ) {
         &logthis('fetch_enrollment_query error: '.$reply.' for '.$dom.' '.$env{'user.name'}.' for '.$queryid.' context: '.$context.' '.$cnum.' maxtries: '.$maxtries.' tries: '.$tries);
     } else {
         my @responses = split(/:/,$reply);
-        if ($homeserver eq $perlvar{'lonHostID'}) {
+        if (grep { $_ eq $homeserver } &current_machine_ids()) {
             foreach my $line (@responses) {
                 my ($key,$value) = split(/=/,$line,2);
                 $$replyref{$key} = $value;
@@ -6474,11 +8431,17 @@ sub fetch_enrollment_query {
 }
 
 sub get_query_reply {
-    my $queryid=shift;
+    my ($queryid,$sleep,$loopmax) = @_;;
+    if (($sleep eq '') || ($sleep !~ /^\d+\.?\d*$/)) {
+        $sleep = 0.2;
+    }
+    if (($loopmax eq '') || ($loopmax =~ /\D/)) {
+        $loopmax = 100;
+    }
     my $replyfile=LONCAPA::tempdir().$queryid;
     my $reply='';
-    for (1..100) {
-	sleep 2;
+    for (1..$loopmax) {
+	sleep($sleep);
         if (-e $replyfile.'.end') {
 	    if (open(my $fh,$replyfile)) {
 		$reply = join('',<$fh>);
@@ -6597,8 +8560,8 @@ sub auto_validate_instcode {
     }
     $response=&unescape(&reply('autovalidateinstcode:'.$cdom.':'.
                         &escape($instcode).':'.&escape($owner),$homeserver));
-    my ($outcome,$description) = map { &unescape($_); } split('&',$response,2);
-    return ($outcome,$description);
+    my ($outcome,$description,$defaultcredits) = map { &unescape($_); } split('&',$response,3);
+    return ($outcome,$description,$defaultcredits);
 }
 
 sub auto_create_password {
@@ -6840,17 +8803,20 @@ sub auto_courserequest_checks {
 }
 
 sub auto_courserequest_validation {
-    my ($dom,$owner,$crstype,$inststatuslist,$instcode,$instseclist) = @_;
+    my ($dom,$owner,$crstype,$inststatuslist,$instcode,$instseclist,$custominfo) = @_;
     my ($homeserver,$response);
     if ($dom =~ /^$match_domain$/) {
         $homeserver = &domain($dom,'primary');
     }
-    unless ($homeserver eq 'no_host') {  
-          
+    unless ($homeserver eq 'no_host') {
+        my $customdata;
+        if (ref($custominfo) eq 'HASH') {
+            $customdata = &freeze_escape($custominfo);
+        }
         $response=&unescape(&reply('autocrsreqvalidation:'.$dom.':'.&escape($owner).
                                     ':'.&escape($crstype).':'.&escape($inststatuslist).
-                                    ':'.&escape($instcode).':'.&escape($instseclist),
-                                    $homeserver));
+                                    ':'.&escape($instcode).':'.&escape($instseclist).':'.
+                                    $customdata,$homeserver));
     }
     return $response;
 }
@@ -6869,6 +8835,136 @@ sub auto_validate_class_sec {
     return $response;
 }
 
+sub auto_crsreq_update {
+    my ($cdom,$cnum,$crstype,$action,$ownername,$ownerdomain,$fullname,$title,
+        $code,$accessstart,$accessend,$inbound) = @_;
+    my ($homeserver,%crsreqresponse);
+    if ($cdom =~ /^$match_domain$/) {
+        $homeserver = &domain($cdom,'primary');
+    }
+    unless (($homeserver eq 'no_host') || ($homeserver eq '')) {
+        my $info;
+        if (ref($inbound) eq 'HASH') {
+            $info = &freeze_escape($inbound);
+        }
+        my $response=&reply('autocrsrequpdate:'.$cdom.':'.$cnum.':'.&escape($crstype).
+                            ':'.&escape($action).':'.&escape($ownername).':'.
+                            &escape($ownerdomain).':'.&escape($fullname).':'.
+                            &escape($title).':'.&escape($code).':'.
+                            &escape($accessstart).':'.&escape($accessend).':'.$info,
+                            $homeserver);
+        unless ($response =~ /(con_lost|error|no_such_host|refused)/) {
+            my @items = split(/&/,$response);
+            foreach my $item (@items) {
+                my ($key,$value) = split('=',$item);
+                $crsreqresponse{&unescape($key)} = &thaw_unescape($value);
+            }
+        }
+    }
+    return \%crsreqresponse;
+}
+
+sub auto_export_grades {
+    my ($cdom,$cnum,$inforef,$gradesref) = @_;
+    my ($homeserver,%exportresponse);
+    if ($cdom =~ /^$match_domain$/) {
+        $homeserver = &domain($cdom,'primary');
+    }
+    unless (($homeserver eq 'no_host') || ($homeserver eq '')) {
+        my $info;
+        if (ref($inforef) eq 'HASH') {
+            $info = &freeze_escape($inforef);
+        }
+        if (ref($gradesref) eq 'HASH') {
+            my $grades = &freeze_escape($gradesref);
+            my $response=&reply('encrypt:autoexportgrades:'.$cdom.':'.$cnum.':'.
+                                $info.':'.$grades,$homeserver);
+            unless ($response =~ /(con_lost|error|no_such_host|refused|unknown_command)/) {
+                my @items = split(/&/,$response);
+                foreach my $item (@items) {
+                    my ($key,$value) = split('=',$item);
+                    $exportresponse{&unescape($key)} = &thaw_unescape($value);
+                }
+            }
+        }
+    }
+    return \%exportresponse;
+}
+
+sub check_instcode_cloning {
+    my ($codedefaults,$code_order,$cloner,$clonefromcode,$clonetocode) = @_;
+    unless ((ref($codedefaults) eq 'HASH') && (ref($code_order) eq 'ARRAY')) {
+        return;
+    }
+    my $canclone;
+    if (@{$code_order} > 0) {
+        my $instcoderegexp ='^';
+        my @clonecodes = split(/\&/,$cloner);
+        foreach my $item (@{$code_order}) {
+            if (grep(/^\Q$item\E=/,@clonecodes)) {
+                foreach my $pair (@clonecodes) {
+                    my ($key,$val) = split(/\=/,$pair,2);
+                    $val = &unescape($val);
+                    if ($key eq $item) {
+                        $instcoderegexp .= '('.$val.')';
+                        last;
+                    }
+                }
+            } else {
+                $instcoderegexp .= $codedefaults->{$item};
+            }
+        }
+        $instcoderegexp .= '$';
+        my (@from,@to);
+        eval {
+               (@from) = ($clonefromcode =~ /$instcoderegexp/);
+               (@to) = ($clonetocode =~ /$instcoderegexp/);
+        };
+        if ((@from > 0) && (@to > 0)) {
+            my @diffs = &Apache::loncommon::compare_arrays(\@from,\@to);
+            if (!@diffs) {
+                $canclone = 1;
+            }
+        }
+    }
+    return $canclone;
+}
+
+sub default_instcode_cloning {
+    my ($clonedom,$domdefclone,$clonefromcode,$clonetocode,$codedefaultsref,$codeorderref) = @_;
+    my (%codedefaults,@code_order,$canclone);
+    if ((ref($codedefaultsref) eq 'HASH') && (ref($codeorderref) eq 'ARRAY')) {
+        %codedefaults = %{$codedefaultsref};
+        @code_order = @{$codeorderref};
+    } elsif ($clonedom) {
+        &auto_instcode_defaults($clonedom,\%codedefaults,\@code_order);
+    }
+    if (($domdefclone) && (@code_order)) {
+        my @clonecodes = split(/\+/,$domdefclone);
+        my $instcoderegexp ='^';
+        foreach my $item (@code_order) {
+            if (grep(/^\Q$item\E$/,@clonecodes)) {
+                $instcoderegexp .= '('.$codedefaults{$item}.')';
+            } else {
+                $instcoderegexp .= $codedefaults{$item};
+            }
+        }
+        $instcoderegexp .= '$';
+        my (@from,@to);
+        eval {
+            (@from) = ($clonefromcode =~ /$instcoderegexp/);
+            (@to) = ($clonetocode =~ /$instcoderegexp/);
+        };
+        if ((@from > 0) && (@to > 0)) {
+            my @diffs = &Apache::loncommon::compare_arrays(\@from,\@to);
+            if (!@diffs) {
+                $canclone = 1;
+            }
+        }
+    }
+    return $canclone;
+}
+
 # ------------------------------------------------------- Course Group routines
 
 sub get_coursegroups {
@@ -6964,8 +9060,7 @@ sub get_users_groups {
     } else {  
         $grouplist = '';
         my $courseurl = &courseid_to_courseurl($courseid);
-        my $extra = &freeze_escape({'skipcheck' => 1});
-        my %roleshash = &dump('roles',$udom,$uname,$courseurl,undef,$extra);
+        my %roleshash = &dump('roles',$udom,$uname,$courseurl);
         my $access_end = $env{'course.'.$courseid.
                               '.default_enrollment_end_date'};
         my $now = time;
@@ -7017,6 +9112,7 @@ sub plaintext {
     my %rolenames = (
                       Course    => 'std',
                       Community => 'alt1',
+                      Placement => 'std',
                     );
     if ($cid ne '') {
         if ($env{'course.'.$cid.'.'.$short.'.plaintext'} ne '') {
@@ -7137,6 +9233,41 @@ sub assignrole {
                             }
                         }
                     }
+                } elsif ($context eq 'requestauthor') {
+                    if (($udom eq $env{'user.domain'}) && ($uname eq $env{'user.name'}) && 
+                        ($url eq '/'.$udom.'/') && ($role eq 'au')) {
+                        if ($env{'environment.requestauthor'} eq 'automatic') {
+                            $refused = '';
+                        } else {
+                            my %domdefaults = &get_domain_defaults($udom);
+                            if (ref($domdefaults{'requestauthor'}) eq 'HASH') {
+                                my $checkbystatus;
+                                if ($env{'user.adv'}) { 
+                                    my $disposition = $domdefaults{'requestauthor'}{'_LC_adv'};
+                                    if ($disposition eq 'automatic') {
+                                        $refused = '';
+                                    } elsif ($disposition eq '') {
+                                        $checkbystatus = 1;
+                                    } 
+                                } else {
+                                    $checkbystatus = 1;
+                                }
+                                if ($checkbystatus) {
+                                    if ($env{'environment.inststatus'}) {
+                                        my @inststatuses = split(/,/,$env{'environment.inststatus'});
+                                        foreach my $type (@inststatuses) {
+                                            if (($type ne '') &&
+                                                ($domdefaults{'requestauthor'}{$type} eq 'automatic')) {
+                                                $refused = '';
+                                            }
+                                        }
+                                    } elsif ($domdefaults{'requestauthor'}{'default'} eq 'automatic') {
+                                        $refused = '';
+                                    }
+                                }
+                            }
+                        }
+                    }
                 }
                 if ($refused) {
                     &logthis('Refused assignrole: '.$udom.' '.$uname.' '.$url.
@@ -7186,11 +9317,26 @@ sub assignrole {
 # log new user role if status is ok
     if ($answer eq 'ok') {
 	&userrolelog($role,$uname,$udom,$url,$start,$end);
+        if (($role eq 'cc') || ($role eq 'in') ||
+            ($role eq 'ep') || ($role eq 'ad') ||
+            ($role eq 'ta') || ($role eq 'st') ||
+            ($role=~/^cr/) || ($role eq 'gr') ||
+            ($role eq 'co')) {
 # for course roles, perform group memberships changes triggered by role change.
-        &courserolelog($role,$uname,$udom,$url,$origstart,$origend,$delflag,$selfenroll,$context);
-        unless ($role =~ /^gr/) {
-            &Apache::longroup::group_changes($udom,$uname,$url,$role,$origend,
-                                             $origstart,$selfenroll,$context);
+            unless ($role =~ /^gr/) {
+                &Apache::longroup::group_changes($udom,$uname,$url,$role,$origend,
+                                                 $origstart,$selfenroll,$context);
+            }
+            &courserolelog($role,$uname,$udom,$url,$origstart,$origend,$delflag,
+                           $selfenroll,$context);
+        } elsif (($role eq 'li') || ($role eq 'dg') || ($role eq 'sc') ||
+                 ($role eq 'au') || ($role eq 'dc') || ($role eq 'dh') ||
+                 ($role eq 'da')) {
+            &domainrolelog($role,$uname,$udom,$url,$origstart,$origend,$delflag,
+                           $context);
+        } elsif (($role eq 'ca') || ($role eq 'aa')) {
+            &coauthorrolelog($role,$uname,$udom,$url,$origstart,$origend,$delflag,
+                             $context); 
         }
         if ($role eq 'cc') {
             &autoupdate_coowners($url,$end,$start,$uname,$udom);
@@ -7380,7 +9526,7 @@ sub modifyuser {
                   'current user id "'.$uidhash{$uname}.'".';
           }
        } else {
-	  &idput($udom,($uname => $uid));
+	  &idput($udom,{$uname => $uid},$uhome,'ids');
        }
     }
 # -------------------------------------------------------------- Add names, etc
@@ -7490,7 +9636,7 @@ sub modifyuser {
 sub modifystudent {
     my ($udom,$uname,$uid,$umode,$upass,$first,$middle,$last,$gene,$usec,
         $end,$start,$forceid,$desiredhome,$email,$type,$locktype,$cid,
-        $selfenroll,$context,$inststatus)=@_;
+        $selfenroll,$context,$inststatus,$credits,$instsec)=@_;
     if (!$cid) {
 	unless ($cid=$env{'request.course.id'}) {
 	    return 'not_in_class';
@@ -7502,15 +9648,17 @@ sub modifystudent {
          $desiredhome,$email,$inststatus);
     unless ($reply eq 'ok') { return $reply; }
     # This will cause &modify_student_enrollment to get the uid from the
-    # students environment
+    # student's environment
     $uid = undef if (!$forceid);
     $reply = &modify_student_enrollment($udom,$uname,$uid,$first,$middle,$last,
-					$gene,$usec,$end,$start,$type,$locktype,$cid,$selfenroll,$context);
+                                        $gene,$usec,$end,$start,$type,$locktype,
+                                        $cid,$selfenroll,$context,$credits,$instsec);
     return $reply;
 }
 
 sub modify_student_enrollment {
-    my ($udom,$uname,$uid,$first,$middle,$last,$gene,$usec,$end,$start,$type,$locktype,$cid,$selfenroll,$context) = @_;
+    my ($udom,$uname,$uid,$first,$middle,$last,$gene,$usec,$end,$start,$type,
+        $locktype,$cid,$selfenroll,$context,$credits,$instsec) = @_;
     my ($cdom,$cnum,$chome);
     if (!$cid) {
 	unless ($cid=$env{'request.course.id'}) {
@@ -7557,7 +9705,7 @@ sub modify_student_enrollment {
     my %old_entry = &Apache::lonnet::get('classlist',[$user],$cdom,$cnum);
     my $reply=cput('classlist',
 		   {$user => 
-			join(':',$end,$start,$uid,$usec,$fullname,$type,$locktype) },
+			join(':',$end,$start,$uid,$usec,$fullname,$type,$locktype,$credits,$instsec) },
 		   $cdom,$cnum);
     if (($reply eq 'ok') || ($reply eq 'delayed')) {
         &devalidate_getsection_cache($udom,$uname,$cid);
@@ -7778,13 +9926,16 @@ sub generate_coursenum {
 }
 
 sub is_course {
-    my ($cdom,$cnum) = @_;
-    my %courses = &courseiddump($cdom,'.',1,'.','.',$cnum,undef,
-				undef,'.');
-    if (exists($courses{$cdom.'_'.$cnum})) {
-        return 1;
-    }
-    return 0;
+    my ($cdom, $cnum) = scalar(@_) == 1 ? 
+         ($_[0] =~ /^($match_domain)_($match_courseid)$/)  :  @_;
+
+    return unless $cdom and $cnum;
+
+    my %courses = &courseiddump($cdom, '.', 1, '.', '.', $cnum, undef, undef,
+        '.');
+
+    return unless(exists($courses{$cdom.'_'.$cnum}));
+    return wantarray ? ($cdom, $cnum) : $cdom.'_'.$cnum;
 }
 
 sub store_userdata {
@@ -7808,6 +9959,9 @@ sub store_userdata {
                     $namevalue.=&escape($key).'='.&freeze_escape($$storehash{$key}).'&';
                 }
                 $namevalue=~s/\&$//;
+                unless ($namespace eq 'courserequests') {
+                    $datakey = &escape($datakey);
+                }
                 $result =  &reply("store:$udom:$uname:$namespace:$datakey:".
                                   $namevalue,$uhome);
             }
@@ -7979,6 +10133,90 @@ sub files_not_in_path {
     return (@return_files);
 }
 
+#------------------------------Submitted/Handedback Portfolio Files Versioning
+ 
+sub portfiles_versioning {
+    my ($symb,$domain,$stu_name,$portfiles,$versioned_portfiles) = @_;
+    my $portfolio_root = '/userfiles/portfolio';
+    return unless ((ref($portfiles) eq 'ARRAY') && (ref($versioned_portfiles) eq 'ARRAY'));
+    foreach my $file (@{$portfiles}) {
+        &unmark_as_readonly($domain,$stu_name,[$symb,$env{'request.course.id'}],$file);
+        my ($directory,$answer_file) =($file =~ /^(.*?)([^\/]*)$/);
+        my ($answer_name,$answer_ver,$answer_ext) = &file_name_version_ext($answer_file);
+        my $getpropath = 1;
+        my ($dir_list,$listerror) = &dirlist($portfolio_root.$directory,$domain,
+                                             $stu_name,$getpropath);
+        my $version = &get_next_version($answer_name,$answer_ext,$dir_list);
+        my $new_answer = 
+            &version_selected_portfile($domain,$stu_name,$directory,$answer_file,$version);
+        if ($new_answer ne 'problem getting file') {
+            push(@{$versioned_portfiles}, $directory.$new_answer);
+            &mark_as_readonly($domain,$stu_name,[$directory.$new_answer],
+                              [$symb,$env{'request.course.id'},'graded']);
+        }
+    }
+}
+
+sub get_next_version {
+    my ($answer_name, $answer_ext, $dir_list) = @_;
+    my $version;
+    if (ref($dir_list) eq 'ARRAY') {
+        foreach my $row (@{$dir_list}) {
+            my ($file) = split(/\&/,$row,2);
+            my ($file_name,$file_version,$file_ext) =
+                &file_name_version_ext($file);
+            if (($file_name eq $answer_name) &&
+                ($file_ext eq $answer_ext)) {
+                     # gets here if filename and extension match,
+                     # regardless of version
+                if ($file_version ne '') {
+                    # a versioned file is found  so save it for later
+                    if ($file_version > $version) {
+                        $version = $file_version;
+                    }
+                }
+            }
+        }
+    }
+    $version ++;
+    return($version);
+}
+
+sub version_selected_portfile {
+    my ($domain,$stu_name,$directory,$file_name,$version) = @_;
+    my ($answer_name,$answer_ver,$answer_ext) =
+        &file_name_version_ext($file_name);
+    my $new_answer;
+    $env{'form.copy'} =
+        &getfile("/uploaded/$domain/$stu_name/portfolio$directory$file_name");
+    if($env{'form.copy'} eq '-1') {
+        $new_answer = 'problem getting file';
+    } else {
+        $new_answer = $answer_name.'.'.$version.'.'.$answer_ext;
+        my $copy_result = 
+            &finishuserfileupload($stu_name,$domain,'copy',
+                                  '/portfolio'.$directory.$new_answer);
+    }
+    undef($env{'form.copy'});
+    return ($new_answer);
+}
+
+sub file_name_version_ext {
+    my ($file)=@_;
+    my @file_parts = split(/\./, $file);
+    my ($name,$version,$ext);
+    if (@file_parts > 1) {
+        $ext=pop(@file_parts);
+        if (@file_parts > 1 && $file_parts[-1] =~ /^\d+$/) {
+            $version=pop(@file_parts);
+        }
+        $name=join('.',@file_parts);
+    } else {
+        $name=join('.',@file_parts);
+    }
+    return($name,$version,$ext);
+}
+
 #----------------------------------------------Get portfolio file permissions
 
 sub get_portfile_permissions {
@@ -8075,9 +10313,9 @@ sub modify_access_controls {
     my $tries = 0;
     my $gotlock = &newput('file_permissions',$lockhash,$domain,$user);
    
-    while (($gotlock ne 'ok') && $tries <3) {
+    while (($gotlock ne 'ok') && $tries < 10) {
         $tries ++;
-        sleep 1;
+        sleep(0.1);
         $gotlock = &newput('file_permissions',$lockhash,$domain,$user);
     }
     if ($gotlock eq 'ok') {
@@ -8123,49 +10361,132 @@ sub modify_access_controls {
 }
 
 sub make_public_indefinitely {
-    my ($requrl) = @_;
+    my (@requrl) = @_;
+    return &automated_portfile_access('public',\@requrl);
+}
+
+sub automated_portfile_access {
+    my ($accesstype,$addsref,$delsref,$info) = @_;
+    unless (($accesstype eq 'public') || ($accesstype eq 'ip')) {
+        return 'invalid';
+    }
+    my %urls;
+    if (ref($addsref) eq 'ARRAY') {
+        foreach my $requrl (@{$addsref}) {
+            if (&is_portfolio_url($requrl)) {
+                unless (exists($urls{$requrl})) {
+                    $urls{$requrl} = 'add';
+                }
+            }
+        }
+    }
+    if (ref($delsref) eq 'ARRAY') {
+        foreach my $requrl (@{$delsref}) { 
+            if (&is_portfolio_url($requrl)) {
+                unless (exists($urls{$requrl})) {
+                    $urls{$requrl} = 'delete'; 
+                }
+            }
+        }
+    }
+    unless (keys(%urls)) {
+        return 'invalid';
+    }
+    my $ip;
+    if ($accesstype eq 'ip') {
+        if (ref($info) eq 'HASH') {
+            if ($info->{'ip'} ne '') {
+                $ip = $info->{'ip'};
+            }
+        }
+        if ($ip eq '') {
+            return 'invalid';
+        }
+    }
+    my $errors;
     my $now = time;
-    my $action = 'activate';
-    my $aclnum = 0;
-    if (&is_portfolio_url($requrl)) {
+    my %current_perms;
+    foreach my $requrl (sort(keys(%urls))) {
+        my $action;
+        if ($urls{$requrl} eq 'add') {
+            $action = 'activate';
+        } else {
+            $action = 'none';
+        }
+        my $aclnum = 0;
         my (undef,$udom,$unum,$file_name,$group) =
             &parse_portfolio_url($requrl);
-        my $current_perms = &get_portfile_permissions($udom,$unum);
-        my %access_controls = &get_access_controls($current_perms,
+        unless (exists($current_perms{$unum.':'.$udom})) {
+            $current_perms{$unum.':'.$udom} = &get_portfile_permissions($udom,$unum);
+        }
+        my %access_controls = &get_access_controls($current_perms{$unum.':'.$udom},
                                                    $group,$file_name);
         foreach my $key (keys(%{$access_controls{$file_name}})) {
             my ($num,$scope,$end,$start) = 
                 ($key =~ /^([^:]+):([a-z]+)_(\d*)_?(\d*)$/);
-            if ($scope eq 'public') {
-                if ($start <= $now && $end == 0) {
-                    $action = 'none';
-                } else {
+            if ($scope eq $accesstype) {
+                if (($start <= $now) && ($end == 0)) {
+                    if ($accesstype eq 'ip') {
+                        if (ref($access_controls{$file_name}{$key}) eq 'HASH') {
+                            if (ref($access_controls{$file_name}{$key}{'ip'}) eq 'ARRAY') {
+                                if (grep(/^\Q$ip\E$/,@{$access_controls{$file_name}{$key}{'ip'}})) {
+                                    if ($urls{$requrl} eq 'add') {
+                                        $action = 'none';
+                                        last;
+                                    } else {
+                                        $action = 'delete';
+                                        $aclnum = $num;
+                                        last;
+                                    }
+                                }
+                            }
+                        }
+                    } elsif ($accesstype eq 'public') {
+                        if ($urls{$requrl} eq 'add') {
+                            $action = 'none';
+                            last;
+                        } else {
+                            $action = 'delete';
+                            $aclnum = $num;
+                            last;
+                        }
+                    }
+                } elsif ($accesstype eq 'public') {
                     $action = 'update';
                     $aclnum = $num;
+                    last;
                 }
-                last;
             }
         }
         if ($action eq 'none') {
-             return 'ok';
+            next;
         } else {
             my %changes;
             my $newend = 0;
             my $newstart = $now;
-            my $newkey = $aclnum.':public_'.$newend.'_'.$newstart;
+            my $newkey = $aclnum.':'.$accesstype.'_'.$newend.'_'.$newstart;
             $changes{$action}{$newkey} = {
-                type => 'public',
+                type => $accesstype,
                 time => {
                     start => $newstart,
                     end   => $newend,
                 },
             };
+            if ($accesstype eq 'ip') {
+                $changes{$action}{$newkey}{'ip'} = [$ip];
+            }
             my ($outcome,$deloutcome,$new_values,$translation) =
                 &modify_access_controls($file_name,\%changes,$udom,$unum);
-            return $outcome;
+            unless ($outcome eq 'ok') {
+                $errors .= $outcome.' ';
+            }
         }
+    }
+    if ($errors) {
+        $errors =~ s/\s$//;
+        return $errors;
     } else {
-        return 'invalid';
+        return 'ok';
     }
 }
 
@@ -8370,7 +10691,24 @@ sub dirlist {
             foreach my $user (sort(keys(%allusers))) {
                 push(@alluserslist,$user.'&user');
             }
-            return (\@alluserslist);
+
+            if (!%listerror) {
+                # no errors
+                return (\@alluserslist);
+            } elsif (scalar(keys(%servers)) == 1) {
+                # one library server, one error 
+                my ($key) = keys(%listerror);
+                return (\@alluserslist, $listerror{$key});
+            } elsif ( grep { $_ eq 'con_lost' } values(%listerror) ) {
+                # con_lost indicates that we might miss data from at least one
+                # library server
+                return (\@alluserslist, 'con_lost');
+            } else {
+                # multiple library servers and no con_lost -> data should be
+                # complete. 
+                return (\@alluserslist);
+            }
+
         } else {
             return ([],'missing username');
         }
@@ -8443,6 +10781,115 @@ sub stat_file {
     return ();
 }
 
+# --------------------------------------------------------- recursedirs
+# Recursive function to traverse either a specific user's Authoring Space
+# or corresponding Published Resource Space, and populate the hash ref:
+# $dirhashref with URLs of all directories, and if $filehashref hash
+# ref arg is provided, the URLs of any files, excluding versioned, .meta,
+# or .rights files in resource space, and .meta, .save, .log, and .bak
+# files in Authoring Space.
+#
+# Inputs:
+#
+# $is_home - true if current server is home server for user's space
+# $context - either: priv, or res respectively for Authoring or Resource Space.
+# $docroot - Document root (i.e., /home/httpd/html
+# $toppath - Top level directory (i.e., /res/$dom/$uname or /priv/$dom/$uname
+# $relpath - Current path (relative to top level).
+# $dirhashref - reference to hash to populate with URLs of directories (Required)
+# $filehashref - reference to hash to populate with URLs of files (Optional)
+#
+# Returns: nothing
+#
+# Side Effects: populates $dirhashref, and $filehashref (if provided).
+#
+# Currently used by interface/londocs.pm to create linked select boxes for
+# directory and filename to import a Course "Author" resource into a course, and
+# also to create linked select boxes for Authoring Space and Directory to choose
+# save location for creation of a new "standard" problem from the Course Editor.
+#
+
+sub recursedirs {
+    my ($is_home,$context,$docroot,$toppath,$relpath,$dirhashref,$filehashref) = @_;
+    return unless (ref($dirhashref) eq 'HASH');
+    my $currpath = $docroot.$toppath;
+    if ($relpath) {
+        $currpath .= "/$relpath";
+    }
+    my $savefile;
+    if (ref($filehashref)) {
+        $savefile = 1;
+    }
+    if ($is_home) {
+        if (opendir(my $dirh,$currpath)) {
+            foreach my $item (sort { lc($a) cmp lc($b) } grep(!/^\.+$/,readdir($dirh))) {
+                next if ($item eq '');
+                if (-d "$currpath/$item") {
+                    my $newpath;
+                    if ($relpath) {
+                        $newpath = "$relpath/$item";
+                    } else {
+                        $newpath = $item;
+                    }
+                    $dirhashref->{&Apache::lonlocal::js_escape($newpath)} = 1;
+                    &recursedirs($is_home,$context,$docroot,$toppath,$newpath,$dirhashref,$filehashref);
+                } elsif ($savefile) {
+                    if ($context eq 'priv') {
+                        unless ($item =~ /\.(meta|save|log|bak|DS_Store)$/) {
+                            $filehashref->{&Apache::lonlocal::js_escape($relpath)}{$item} = 1;
+                        }
+                    } else {
+                        unless (($item =~ /\.meta$/) || ($item =~ /\.\d+\.\w+$/) || ($item =~ /\.rights$/)) {
+                            $filehashref->{&Apache::lonlocal::js_escape($relpath)}{$item} = 1;
+                        }
+                    }
+                }
+            }
+            closedir($dirh);
+        }
+    } else {
+        my ($dirlistref,$listerror) =
+            &dirlist($toppath.$relpath);
+        my @dir_lines;
+        my $dirptr=16384;
+        if (ref($dirlistref) eq 'ARRAY') {
+            foreach my $dir_line (sort
+                              {
+                                  my ($afile)=split('&',$a,2);
+                                  my ($bfile)=split('&',$b,2);
+                                  return (lc($afile) cmp lc($bfile));
+                              } (@{$dirlistref})) {
+                my ($item,$dom,undef,$testdir,undef,undef,undef,undef,$size,undef,$mtime,undef,undef,undef,$obs,undef) =
+                    split(/\&/,$dir_line,16);
+                $item =~ s/\s+$//;
+                next if (($item =~ /^\.\.?$/) || ($obs));
+                if ($dirptr&$testdir) {
+                    my $newpath;
+                    if ($relpath) {
+                        $newpath = "$relpath/$item";
+                    } else {
+                        $relpath = '/';
+                        $newpath = $item;
+                    }
+                    $dirhashref->{&Apache::lonlocal::js_escape($newpath)} = 1;
+                    &recursedirs($is_home,$context,$docroot,$toppath,$newpath,$dirhashref,$filehashref);
+                } elsif ($savefile) {
+                    if ($context eq 'priv') {
+                        unless ($item =~ /\.(meta|save|log|bak|DS_Store)$/) {
+                            $filehashref->{$relpath}{$item} = 1;
+                        }
+                    } else {
+                        unless (($item =~ /\.meta$/) || ($item =~ /\.\d+\.\w+$/)) {
+                            $filehashref->{$relpath}{$item} = 1;
+                        }
+                    }
+                }
+            }
+        }
+    }
+    return;
+}
+
 # -------------------------------------------------------- Value of a Condition
 
 # gets the value of a specific preevaluated condition
@@ -8587,10 +11034,12 @@ sub get_userresdata {
     }
     #error 2 occurs when the .db doesn't exist
     if ($tmp!~/error: 2 /) {
-	&logthis("<font color=\"blue\">WARNING:".
-		 " Trying to get resource data for ".
-		 $uname." at ".$udom.": ".
-		 $tmp."</font>");
+        if ((!defined($cached)) || ($tmp ne 'con_lost')) {
+	    &logthis("<font color=\"blue\">WARNING:".
+		     " Trying to get resource data for ".
+		     $uname." at ".$udom.": ".
+		     $tmp."</font>");
+        }
     } elsif ($tmp=~/error: 2 /) {
 	#&EXT_cache_set($udom,$uname);
 	&do_cache_new('userres',$hashid,undef,600);
@@ -8604,7 +11053,14 @@ sub get_userresdata {
 #  Parameters:
 #     $name      - Course/user name.
 #     $domain    - Name of the domain the user/course is registered on.
-#     $type      - Type of thing $name is (must be 'course' or 'user'
+#     $type      - Type of thing $name is (must be 'course' or 'user')
+#     $mapp      - decluttered URL of enclosing map  
+#     $recursed  - Ref to scalar -- set to 1, if nested maps have been recursed.
+#     $recurseup - Ref to array of map URLs, starting with map containing
+#                  $mapp up through hierarchy of nested maps to top level map.  
+#     $courseid  - CourseID (first part of param identifier).
+#     $modifier  - Middle part of param identifier.
+#     $what      - Last part of param identifier.
 #     @which     - Array of names of resources desired.
 #  Returns:
 #     The value of the first reasource in @which that is found in the
@@ -8614,7 +11070,8 @@ sub get_userresdata {
 #     'user', an undefined  reference is returned.
 #     If none of the resources are found, an undef is returned
 sub resdata {
-    my ($name,$domain,$type,@which)=@_;
+    my ($name,$domain,$type,$mapp,$recursed,$recurseup,$courseid,
+        $modifier,$what,@which)=@_;
     my $result;
     if ($type eq 'course') {
 	$result=&get_courseresdata($name,$domain);
@@ -8623,17 +11080,90 @@ sub resdata {
     }
     if (!ref($result)) { return $result; }    
     foreach my $item (@which) {
-	if (defined($result->{$item->[0]})) {
+        if ($item->[1] eq 'course') {
+            if ((ref($recurseup) eq 'ARRAY') && (ref($recursed) eq 'SCALAR')) {
+                unless ($$recursed) {
+                    @{$recurseup} = &get_map_hierarchy($mapp,$courseid);
+                    $$recursed = 1;
+                }
+                foreach my $item (@${recurseup}) {
+                    my $norecursechk=$courseid.$modifier.$item.'___(all).'.$what;
+                    last if (defined($result->{$norecursechk}));
+                    my $recursechk=$courseid.$modifier.$item.'___(rec).'.$what;
+                    if (defined($result->{$recursechk})) { return [$result->{$recursechk},'map']; }
+                }
+            }
+        }
+        if (defined($result->{$item->[0]})) {
 	    return [$result->{$item->[0]},$item->[1]];
 	}
     }
     return undef;
 }
 
+sub get_domain_ltitools {
+    my ($cdom) = @_;
+    my %ltitools;
+    my ($result,$cached)=&is_cached_new('ltitools',$cdom);
+    if (defined($cached)) {
+        if (ref($result) eq 'HASH') {
+            %ltitools = %{$result};
+        }
+    } else {
+        my %domconfig = &get_dom('configuration',['ltitools'],$cdom);
+        if (ref($domconfig{'ltitools'}) eq 'HASH') {
+            %ltitools = %{$domconfig{'ltitools'}};
+            my %encdomconfig = &get_dom('encconfig',['ltitools'],$cdom);
+            if (ref($encdomconfig{'ltitools'}) eq 'HASH') {
+                foreach my $id (keys(%ltitools)) {
+                    if (ref($encdomconfig{'ltitools'}{$id}) eq 'HASH') {
+                        foreach my $item ('key','secret') {
+                            $ltitools{$id}{$item} = $encdomconfig{'ltitools'}{$id}{$item};
+                        }
+                    }
+                }
+            }
+        }
+        my $cachetime = 24*60*60;
+        &do_cache_new('ltitools',$cdom,\%ltitools,$cachetime);
+    }
+    return %ltitools;
+}
+
+sub get_numsuppfiles {
+    my ($cnum,$cdom,$ignorecache)=@_;
+    my $hashid=$cnum.':'.$cdom;
+    my ($suppcount,$cached);
+    unless ($ignorecache) {
+        ($suppcount,$cached) = &is_cached_new('suppcount',$hashid);
+    }
+    unless (defined($cached)) {
+        my $chome=&homeserver($cnum,$cdom);
+        unless ($chome eq 'no_host') {
+            ($suppcount,my $errors) = (0,0);
+            my $suppmap = 'supplemental.sequence';
+            ($suppcount,$errors) = 
+                &Apache::loncommon::recurse_supplemental($cnum,$cdom,$suppmap,$suppcount,$errors);
+        }
+        &do_cache_new('suppcount',$hashid,$suppcount,600);
+    }
+    return $suppcount;
+}
+
 #
 # EXT resource caching routines
 #
 
+{
+# Cache (5 seconds) of map hierarchy for speedup of navmaps display
+#
+# The course for which we cache
+my $cachedmapkey='';
+# The cached recursive maps for this course
+my %cachedmaps=();
+# When this was last done
+my $cachedmaptime='';
+
 sub clear_EXT_cache_status {
     &delenv('cache.EXT.');
 }
@@ -8658,7 +11188,7 @@ sub EXT_cache_set {
 # --------------------------------------------------------- Value of a Variable
 sub EXT {
 
-    my ($varname,$symbparm,$udom,$uname,$usection,$recurse)=@_;
+    my ($varname,$symbparm,$udom,$uname,$usection,$recurse,$cid)=@_;
     unless ($varname) { return ''; }
     #get real user name/domain, courseid and symb
     my $courseid;
@@ -8773,36 +11303,60 @@ sub EXT {
 	    if (!$symbparm) { $symbparm=&symbread(); }
 	}
 
-	if ($space eq 'title') {
-	    if (!$symbparm) { $symbparm = $env{'request.filename'}; }
-	    return &gettitle($symbparm);
-	}
+        if ($qualifier eq '') {
+	    if ($space eq 'title') {
+	        if (!$symbparm) { $symbparm = $env{'request.filename'}; }
+	        return &gettitle($symbparm);
+	    }
 	
-	if ($space eq 'map') {
-	    my ($map) = &decode_symb($symbparm);
-	    return &symbread($map);
-	}
-	if ($space eq 'filename') {
-	    if ($symbparm) {
-		return &clutter((&decode_symb($symbparm))[2]);
+	    if ($space eq 'map') {
+	        my ($map) = &decode_symb($symbparm);
+	        return &symbread($map);
+	    }
+            if ($space eq 'maptitle') {
+                my ($map) = &decode_symb($symbparm);
+                return &gettitle($map);
+            }
+	    if ($space eq 'filename') {
+	        if ($symbparm) {
+		    return &clutter((&decode_symb($symbparm))[2]);
+	        }
+	        return &hreflocation('',$env{'request.filename'});
 	    }
-	    return &hreflocation('',$env{'request.filename'});
-	}
 
-	my ($section, $group, @groups);
-	my ($courselevelm,$courselevel);
-	if ($symbparm && defined($courseid) && 
-	    $courseid eq $env{'request.course.id'}) {
+            if ((defined($courseid)) && ($courseid eq $env{'request.course.id'}) && $symbparm) {
+                if ($space eq 'visibleparts') {
+                    my $navmap = Apache::lonnavmaps::navmap->new();
+                    my $item;
+                    if (ref($navmap)) {
+                        my $res = $navmap->getBySymb($symbparm);
+                        my $parts = $res->parts();
+                        if (ref($parts) eq 'ARRAY') {
+                            $item = join(',',@{$parts});
+                        }
+                        undef($navmap);
+                    }
+                    return $item;
+                }
+            }
+        }
+
+	my ($section, $group, @groups, @recurseup, $recursed);
+	my ($courselevelm,$courseleveli,$courselevel,$mapp);
+        if (($courseid eq '') && ($cid)) {
+            $courseid = $cid;
+        }
+	if (($symbparm && $courseid) && 
+	    (($courseid eq $env{'request.course.id'}) || ($courseid eq $cid)))  {
 
 	    #print '<br>'.$space.' - '.$qualifier.' - '.$spacequalifierrest;
 
 # ----------------------------------------------------- Cascading lookup scheme
 	    my $symbp=$symbparm;
-	    my $mapp=&deversion((&decode_symb($symbp))[0]);
-
+	    $mapp=&deversion((&decode_symb($symbp))[0]);
 	    my $symbparm=$symbp.'.'.$spacequalifierrest;
+            my $recurseparm=$mapp.'___(rec).'.$spacequalifierrest;
 	    my $mapparm=$mapp.'___(all).'.$spacequalifierrest;
-
 	    if (($env{'user.name'} eq $uname) &&
 		($env{'user.domain'} eq $udom)) {
 		$section=$env{'request.course.sec'};
@@ -8819,17 +11373,21 @@ sub EXT {
 
 	    my $seclevel=$courseid.'.['.$section.'].'.$spacequalifierrest;
 	    my $seclevelr=$courseid.'.['.$section.'].'.$symbparm;
+            my $secleveli=$courseid.'.['.$section.'].'.$recurseparm;
 	    my $seclevelm=$courseid.'.['.$section.'].'.$mapparm;
 
 	    $courselevel=$courseid.'.'.$spacequalifierrest;
 	    my $courselevelr=$courseid.'.'.$symbparm;
+            $courseleveli=$courseid.'.'.$recurseparm;
 	    $courselevelm=$courseid.'.'.$mapparm;
 
 # ----------------------------------------------------------- first, check user
 
-	    my $userreply=&resdata($uname,$udom,'user',
+	    my $userreply=&resdata($uname,$udom,'user',$mapp,\$recursed,
+                                   \@recurseup,$courseid,'.',$spacequalifierrest, 
 				       ([$courselevelr,'resource'],
 					[$courselevelm,'map'     ],
+                                        [$courseleveli,'map'     ],
 					[$courselevel, 'course'  ]));
 	    if (defined($userreply)) { return &get_reply($userreply); }
 
@@ -8837,15 +11395,18 @@ sub EXT {
             my $coursereply;
             if (@groups > 0) {
                 $coursereply = &check_group_parms($courseid,\@groups,$symbparm,
-                                       $mapparm,$spacequalifierrest);
-                if (defined($coursereply)) { return &get_reply($coursereply); }
+                                       $recurseparm,$mapparm,$spacequalifierrest,
+                                       $mapp,\$recursed,\@recurseup);
+                if (defined($coursereply)) { return &get_reply($coursereply); } 
             }
 
 	    $coursereply=&resdata($env{'course.'.$courseid.'.num'},
 				  $env{'course.'.$courseid.'.domain'},
-				  'course',
+				  'course',$mapp,\$recursed,\@recurseup,
+                                  $courseid,'.['.$section.'].',$spacequalifierrest,
 				  ([$seclevelr,   'resource'],
 				   [$seclevelm,   'map'     ],
+                                   [$secleveli,   'map'     ],
 				   [$seclevel,    'course'  ],
 				   [$courselevelr,'resource']));
 	    if (defined($coursereply)) { return &get_reply($coursereply); }
@@ -8862,8 +11423,9 @@ sub EXT {
 	    if ($thisparm) { return &get_reply([$thisparm,'resource']); }
 	}
 # ------------------------------------------ fourth, look in resource metadata
-
-	$spacequalifierrest=~s/\./\_/;
+ 
+        my $what = $spacequalifierrest;
+	$what=~s/\./\_/;
 	my $filename;
 	if (!$symbparm) { $symbparm=&symbread(); }
 	if ($symbparm) {
@@ -8871,18 +11433,20 @@ sub EXT {
 	} else {
 	    $filename=$env{'request.filename'};
 	}
-	my $metadata=&metadata($filename,$spacequalifierrest);
+	my $metadata=&metadata($filename,$what);
 	if (defined($metadata)) { return &get_reply([$metadata,'resource']); }
-	$metadata=&metadata($filename,'parameter_'.$spacequalifierrest);
+	$metadata=&metadata($filename,'parameter_'.$what);
 	if (defined($metadata)) { return &get_reply([$metadata,'resource']); }
 
-# ---------------------------------------------- fourth, look in rest of course
+# ----------------------------------------------- fifth, look in rest of course
 	if ($symbparm && defined($courseid) && 
 	    $courseid eq $env{'request.course.id'}) {
 	    my $coursereply=&resdata($env{'course.'.$courseid.'.num'},
 				     $env{'course.'.$courseid.'.domain'},
-				     'course',
+				     'course',$mapp,\$recursed,\@recurseup,
+                                     $courseid,'.',$spacequalifierrest,
 				     ([$courselevelm,'map'   ],
+                                      [$courseleveli,'map'   ],
 				      [$courselevel, 'course']));
 	    if (defined($coursereply)) { return &get_reply($coursereply); }
 	}
@@ -8939,22 +11503,51 @@ sub get_reply {
 }
 
 sub check_group_parms {
-    my ($courseid,$groups,$symbparm,$mapparm,$what) = @_;
-    my @groupitems = ();
-    my $resultitem;
-    my @levels = ([$symbparm,'resource'],[$mapparm,'map'],[$what,'course']);
+    my ($courseid,$groups,$symbparm,$recurseparm,$mapparm,$what,$mapp,
+        $recursed,$recurseupref) = @_;
+    my @levels = ([$symbparm,'resource'],[$mapparm,'map'],[$recurseparm,'map'],
+                  [$what,'course']);
+    my $coursereply;
     foreach my $group (@{$groups}) {
+        my @groupitems = ();
         foreach my $level (@levels) {
              my $item = $courseid.'.['.$group.'].'.$level->[0];
              push(@groupitems,[$item,$level->[1]]);
         }
+        my $coursereply = &resdata($env{'course.'.$courseid.'.num'},
+                                   $env{'course.'.$courseid.'.domain'},
+                                   'course',$mapp,$recursed,$recurseupref,
+                                   $courseid,'.['.$group.'].',$what,
+                                   @groupitems);
+        last if (defined($coursereply));
     }
-    my $coursereply = &resdata($env{'course.'.$courseid.'.num'},
-                            $env{'course.'.$courseid.'.domain'},
-                                     'course',@groupitems);
     return $coursereply;
 }
 
+sub get_map_hierarchy {
+    my ($mapname,$courseid) = @_;
+    my @recurseup = ();
+    if ($mapname) {
+        if (($cachedmapkey eq $courseid) &&
+            (abs($cachedmaptime-time)<5)) {
+            if (ref($cachedmaps{$mapname}) eq 'ARRAY') {
+                return @{$cachedmaps{$mapname}};
+            }
+        }
+        my $navmap = Apache::lonnavmaps::navmap->new();
+        if (ref($navmap)) {
+            @recurseup = $navmap->recurseup_maps($mapname);
+            undef($navmap);
+            $cachedmaps{$mapname} = \@recurseup;
+            $cachedmaptime=time;
+            $cachedmapkey=$courseid;
+        }
+    }
+    return @recurseup;
+}
+
+}
+
 sub sort_course_groups { # Sort groups based on defined rankings. Default is sort().
     my ($courseid,@groups) = @_;
     @groups = sort(@groups);
@@ -9039,11 +11632,11 @@ sub metadata {
     # if it is a non metadata possible uri return quickly
     if (($uri eq '') || 
 	(($uri =~ m|^/*adm/|) && 
-	     ($uri !~ m|^adm/includes|) && ($uri !~ m|/bulletinboard$|)) ||
+	     ($uri !~ m|^adm/includes|) && ($uri !~ m{/(smppg|bulletinboard|ext\.tool)$})) ||
         ($uri =~ m|/$|) || ($uri =~ m|/.meta$|) || ($uri =~ m{^/*uploaded/.+\.sequence$})) {
 	return undef;
     }
-    if (($uri =~ /^priv/ || $uri=~/home\/httpd\/html\/priv/) 
+    if (($uri =~ /^priv/ || $uri=~m{^home/httpd/html/priv}) 
 	&& &Apache::lonxml::get_state('target') =~ /^(|meta)$/) {
 	return undef;
     }
@@ -9304,7 +11897,7 @@ sub metadata {
 
 	$metaentry{':keys'} = join(',',keys(%metathesekeys));
 	&metadata_generate_part0(\%metathesekeys,\%metaentry,$uri);
-	$metaentry{':allpossiblekeys'}=join(',',keys %metathesekeys);
+	$metaentry{':allpossiblekeys'}=join(',',keys(%metathesekeys));
 	&do_cache_new('meta',$uri,\%metaentry,$cachetime);
 # this is the end of "was not already recently cached
     }
@@ -9404,6 +11997,10 @@ sub gettitle {
 	}
 	$title=~s/\&colon\;/\:/gs;
 	if ($title) {
+# Remember both $symb and $title for dynamic metadata
+            $accesshash{$symb.'___crstitle'}=$title;
+            $accesshash{&declutter($map).'___'.&declutter($url).'___usage'}=time;
+# Cache this title and then return it
 	    return &do_cache_new('title',$key,$title,600);
 	}
 	$urlsymb=$url;
@@ -9466,7 +12063,7 @@ sub get_course_slots {
         my %slots=&Apache::lonnet::dump('slots',$cdom,$cnum);
         my ($tmp) = keys(%slots);
         if ($tmp !~ /^(con_lost|error|no_such_host)/i) {
-            &Apache::lonnet::do_cache_new('allslots',$hashid,\%slots,600);
+            &do_cache_new('allslots',$hashid,\%slots,600);
             return %slots;
         }
     }
@@ -9479,6 +12076,41 @@ sub devalidate_slots_cache {
     &devalidate_cache_new('allslots',$hashid);
 }
 
+sub get_coursechange {
+    my ($cdom,$cnum) = @_;
+    if ($cdom eq '' || $cnum eq '') {
+        return unless ($env{'request.course.id'});
+        $cnum = $env{'course.'.$env{'request.course.id'}.'.num'};
+        $cdom = $env{'course.'.$env{'request.course.id'}.'.domain'};
+    }
+    my $hashid=$cdom.'_'.$cnum;
+    my ($change,$cached)=&is_cached_new('crschange',$hashid);
+    if ((defined($cached)) && ($change ne '')) {
+        return $change;
+    } else {
+        my %crshash;
+        %crshash = &get('environment',['internal.contentchange'],$cdom,$cnum);
+        if ($crshash{'internal.contentchange'} eq '') {
+            $change = $env{'course.'.$cdom.'_'.$cnum.'.internal.created'};
+            if ($change eq '') {
+                %crshash = &get('environment',['internal.created'],$cdom,$cnum);
+                $change = $crshash{'internal.created'};
+            }
+        } else {
+            $change = $crshash{'internal.contentchange'};
+        }
+        my $cachetime = 600;
+        &do_cache_new('crschange',$hashid,$change,$cachetime);
+    }
+    return $change;
+}
+
+sub devalidate_coursechange_cache {
+    my ($cnum,$cdom)=@_;
+    my $hashid=$cnum.':'.$cdom;
+    &devalidate_cache_new('crschange',$hashid);
+}
+
 # ------------------------------------------------- Update symbolic store links
 
 sub symblist {
@@ -9506,7 +12138,7 @@ sub symblist {
 # --------------------------------------------------------------- Verify a symb
 
 sub symbverify {
-    my ($symb,$thisurl)=@_;
+    my ($symb,$thisurl,$encstate)=@_;
     my $thisfn=$thisurl;
     $thisfn=&declutter($thisfn);
 # direct jump to resource in page or to a sequence - will construct own symbs
@@ -9525,28 +12157,43 @@ sub symbverify {
 
     if (tie(%bighash,'GDBM_File',$env{'request.course.fn'}.'.db',
                             &GDBM_READER(),0640)) {
+        my $noclutter;
         if (($thisurl =~ m{^/adm/wrapper/ext/}) || ($thisurl =~ m{^ext/})) {
             $thisurl =~ s/\?.+$//;
+            if ($map =~ m{^uploaded/.+\.page$}) {
+                $thisurl =~ s{^(/adm/wrapper|)/ext/}{http://};
+                $thisurl =~ s{^\Qhttp://https://\E}{https://};
+                $noclutter = 1;
+            }
+        }
+        my $ids;
+        if ($noclutter) {
+            $ids=$bighash{'ids_'.$thisurl};
+        } else {
+            $ids=$bighash{'ids_'.&clutter($thisurl)};
         }
-        my $ids=$bighash{'ids_'.&clutter($thisurl)};
         unless ($ids) {
             my $idkey = 'ids_'.($thisurl =~ m{^/}? '' : '/').$thisurl;  
             $ids=$bighash{$idkey};
         }
         if ($ids) {
 # ------------------------------------------------------------------- Has ID(s)
+            if ($thisfn =~ m{^/adm/wrapper/ext/}) {
+                $symb =~ s/\?.+$//;
+            }
 	    foreach my $id (split(/\,/,$ids)) {
 	       my ($mapid,$resid)=split(/\./,$id);
-               if ($thisfn =~ m{^/adm/wrapper/ext/}) {
-                   $symb =~ s/\?.+$//;
-               }
                if (
   &symbclean(&declutter($bighash{'map_id_'.$mapid}).'___'.$resid.'___'.$thisfn)
-   eq $symb) { 
+   eq $symb) {
+                   if (ref($encstate)) {
+                       $$encstate = $bighash{'encrypted_'.$id};
+                   }
 		   if (($env{'request.role.adv'}) ||
 		       ($bighash{'encrypted_'.$id} eq $env{'request.enc'}) ||
                        ($thisurl eq '/adm/navmaps')) {
-		       $okay=1; 
+		       $okay=1;
+                       last;
 		   }
 	       }
 	   }
@@ -9621,9 +12268,15 @@ sub deversion {
 # ------------------------------------------------------ Return symb list entry
 
 sub symbread {
-    my ($thisfn,$donotrecurse)=@_;
+    my ($thisfn,$donotrecurse,$ignorecachednull,$checkforblock,$possibles)=@_;
     my $cache_str='request.symbread.cached.'.$thisfn;
-    if (defined($env{$cache_str})) { return $env{$cache_str}; }
+    if (defined($env{$cache_str})) {
+        if ($ignorecachednull) {
+            return $env{$cache_str} unless ($env{$cache_str} eq '');
+        } else {
+            return $env{$cache_str};
+        }
+    }
 # no filename provided? try from environment
     unless ($thisfn) {
         if ($env{'request.symb'}) {
@@ -9685,18 +12338,46 @@ sub symbread {
 		     my ($mapid,$resid)=split(/\./,$ids);
 		     $syval=&encode_symb($bighash{'map_id_'.$mapid},
 						    $resid,$thisfn);
-                 } elsif (!$donotrecurse) {
+                     if (ref($possibles) eq 'HASH') {
+                         $possibles->{$syval} = 1;    
+                     }
+                     if ($checkforblock) {
+                         my @blockers = &has_comm_blocking('bre',$syval,$bighash{'src_'.$ids});
+                         if (@blockers) {
+                             $syval = '';
+                             return;
+                         }
+                     }
+                 } elsif ((!$donotrecurse) || ($checkforblock) || (ref($possibles) eq 'HASH')) { 
 # ------------------------------------------ There is more than one possibility
                      my $realpossible=0;
                      foreach my $id (@possibilities) {
 			 my $file=$bighash{'src_'.$id};
-                         if (&allowed('bre',$file)) {
-         		    my ($mapid,$resid)=split(/\./,$id);
-                            if ($bighash{'map_type_'.$mapid} ne 'page') {
-				$realpossible++;
-                                $syval=&encode_symb($bighash{'map_id_'.$mapid},
-						    $resid,$thisfn);
-                            }
+                         my $canaccess;
+                         if (($donotrecurse) || ($checkforblock) || (ref($possibles) eq 'HASH')) {
+                             $canaccess = 1;
+                         } else { 
+                             $canaccess = &allowed('bre',$file);
+                         }
+                         if ($canaccess) {
+         		     my ($mapid,$resid)=split(/\./,$id);
+                             if ($bighash{'map_type_'.$mapid} ne 'page') {
+                                 my $poss_syval=&encode_symb($bighash{'map_id_'.$mapid},
+						             $resid,$thisfn);
+                                 if (ref($possibles) eq 'HASH') {
+                                     $possibles->{$syval} = 1;
+                                 }
+                                 if ($checkforblock) {
+                                     my @blockers = &has_comm_blocking('bre',$poss_syval,$file);
+                                     unless (@blockers > 0) {
+                                         $syval = $poss_syval;
+                                         $realpossible++;
+                                     }
+                                 } else {
+                                     $syval = $poss_syval;
+                                     $realpossible++;
+                                 }
+                             }
 			 }
                      }
 		     if ($realpossible!=1) { $syval=''; }
@@ -9704,7 +12385,7 @@ sub symbread {
                      $syval='';
                  }
 	      }
-              untie(%bighash)
+              untie(%bighash);
            }
         }
         if ($syval) {
@@ -10043,8 +12724,12 @@ sub rndseed_CODE_64bit5 {
 sub setup_random_from_rndseed {
     my ($rndseed)=@_;
     if ($rndseed =~/([,:])/) {
-	my ($num1,$num2)=split(/[,:]/,$rndseed);
-	&Math::Random::random_set_seed(abs($num1),abs($num2));
+        my ($num1,$num2) = map { abs($_); } (split(/[,:]/,$rndseed));
+        if ((!$num1) || (!$num2) || ($num1 > 2147483562) || ($num2 > 2147483398)) {
+            &Math::Random::random_set_seed_from_phrase($rndseed);
+        } else {
+            &Math::Random::random_set_seed($num1,$num2);
+        }
     } else {
 	&Math::Random::random_set_seed_from_phrase($rndseed);
     }
@@ -10183,7 +12868,7 @@ sub repcopy_userfile {
     my ($file)=@_;
     my $londocroot = $perlvar{'lonDocRoot'};
     if ($file =~ m{^/*(uploaded|editupload)/}) { $file=&filelocation("",$file); }
-    if ($file =~ m{^\Q$londocroot/lonUsers/\E}) { return 'ok'; }
+    if ($file =~ m{^\Q/home/httpd/lonUsers/\E}) { return 'ok'; }
     my ($cdom,$cnum,$filename) = 
 	($file=~m|^\Q$perlvar{'lonDocRoot'}\E/+userfiles/+($match_domain)/+($match_name)/+(.*)|);
     my $uri="/uploaded/$cdom/$cnum/$filename";
@@ -10223,7 +12908,6 @@ sub repcopy_userfile {
     }
 # now the path exists for sure
 # get a user agent
-    my $ua=new LWP::UserAgent;
     my $transferfile=$file.'.in.transfer';
 # FIXME: this should flock
     if (-e $transferfile) { return 'ok'; }
@@ -10233,7 +12917,7 @@ sub repcopy_userfile {
     my $protocol = $protocol{$homeserver};
     $protocol = 'http' if ($protocol ne 'https');
     $request=new HTTP::Request('GET',$protocol.'://'.&hostname($homeserver).'/raw/'.$uri);
-    my $response=$ua->request($request,$transferfile);
+    my $response = &LONCAPA::LWPReq::makerequest($homeserver,$request,$transferfile,\%perlvar,'',0,1);
 # did it work?
     if ($response->is_error()) {
 	unlink($transferfile);
@@ -10277,9 +12961,8 @@ sub getuploaded {
     my $protocol = $protocol{$homeserver};
     $protocol = 'http' if ($protocol ne 'https');
     $uri = $protocol.'://'.&hostname($homeserver).'/raw/'.$uri;
-    my $ua=new LWP::UserAgent;
     my $request=new HTTP::Request($reqtype,$uri);
-    my $response=$ua->request($request);
+    my $response=&LONCAPA::LWPReq::makerequest($homeserver,$request,'',\%perlvar,'',0,1);
     $$rtncode = $response->code;
     if (! $response->is_success()) {
 	return 'failed';
@@ -10435,11 +13118,14 @@ sub default_login_domain {
 sub declutter {
     my $thisfn=shift;
     if ($thisfn=~m|^/enc/|) { $thisfn=&Apache::lonenc::unencrypted($thisfn); }
-    $thisfn=~s/^\Q$perlvar{'lonDocRoot'}\E//;
+    unless ($thisfn=~m{^/home/httpd/html/priv/}) {
+        $thisfn=~s{^/home/httpd/html}{};
+    }
     $thisfn=~s/^\///;
     $thisfn=~s|^adm/wrapper/||;
     $thisfn=~s|^adm/coursedocs/showdoc/||;
     $thisfn=~s/^res\///;
+    $thisfn=~s/^priv\///;
     unless (($thisfn =~ /^ext/) || ($thisfn =~ /\.(page|sequence)___\d+___ext/)) {
         $thisfn=~s/\?.+$//;
     }
@@ -10477,6 +13163,8 @@ sub clutter {
 #		&logthis("Got a blank emb style");
 	    }
 	}
+    } elsif ($thisfn =~ m{^/adm/$match_domain/$match_courseid/\d+/ext\.tool$}) {
+        $thisfn='/adm/wrapper'.$thisfn;
     }
     return $thisfn;
 }
@@ -10539,12 +13227,12 @@ sub goodbye {
 }
 
 sub get_dns {
-    my ($url,$func,$ignore_cache) = @_;
+    my ($url,$func,$ignore_cache,$nocache,$hashref) = @_;
     if (!$ignore_cache) {
 	my ($content,$cached)=
 	    &Apache::lonnet::is_cached_new('dns',$url);
 	if ($cached) {
-	    &$func($content);
+	    &$func($content,$hashref);
 	    return;
 	}
     }
@@ -10561,16 +13249,16 @@ sub get_dns {
 	$alldns{$host} = $protocol;
     }
     while (%alldns) {
-	my ($dns) = keys(%alldns);
-	my $ua=new LWP::UserAgent;
-        $ua->timeout(30);
+	my ($dns) = sort { $b cmp $a } keys(%alldns);
 	my $request=new HTTP::Request('GET',"$alldns{$dns}://$dns$url");
-	my $response=$ua->request($request);
+        my $response = &LONCAPA::LWPReq::makerequest('',$request,'',\%perlvar,30,0);
         delete($alldns{$dns});
 	next if ($response->is_error());
 	my @content = split("\n",$response->content);
-	&Apache::lonnet::do_cache_new('dns',$url,\@content,30*24*60*60);
-	&$func(\@content);
+	unless ($nocache) {
+	    &do_cache_new('dns',$url,\@content,30*24*60*60);
+	}
+	&$func(\@content,$hashref);
 	return;
     }
     close($config);
@@ -10578,9 +13266,66 @@ sub get_dns {
     &logthis("unable to contact DNS defaulting to on disk file dns_$which.tab\n");
     open($config,"<$perlvar{'lonTabDir'}/dns_$which.tab");
     my @content = <$config>;
-    &$func(\@content);
+    &$func(\@content,$hashref);
+    return;
+}
+
+# ------------------------------------------------------Get DNS checksums file
+sub parse_dns_checksums_tab {
+    my ($lines,$hashref) = @_;
+    my $lonhost = $perlvar{'lonHostID'};
+    my $machine_dom = &Apache::lonnet::host_domain($lonhost);
+    my $loncaparev = &get_server_loncaparev($machine_dom);
+    my $distro = (split(/\:/,&get_server_distarch($lonhost)))[0];
+    my $webconfdir = '/etc/httpd/conf';
+    if ($distro =~ /^(ubuntu|debian)(\d+)$/) {
+        $webconfdir = '/etc/apache2';
+    } elsif ($distro =~ /^sles(\d+)$/) {
+        if ($1 >= 10) {
+            $webconfdir = '/etc/apache2';
+        }
+    } elsif ($distro =~ /^suse(\d+\.\d+)$/) {
+        if ($1 >= 10.0) {
+            $webconfdir = '/etc/apache2';
+        }
+    }
+    my ($release,$timestamp) = split(/\-/,$loncaparev);
+    my (%chksum,%revnum);
+    if (ref($lines) eq 'ARRAY') {
+        chomp(@{$lines});
+        my $version = shift(@{$lines});
+        if ($version eq $release) {  
+            foreach my $line (@{$lines}) {
+                my ($file,$version,$shasum) = split(/,/,$line);
+                if ($file =~ m{^/etc/httpd/conf}) {
+                    if ($webconfdir eq '/etc/apache2') {
+                        $file =~ s{^\Q/etc/httpd/conf/\E}{$webconfdir/};
+                    }
+                }
+                $chksum{$file} = $shasum;
+                $revnum{$file} = $version;
+            }
+            if (ref($hashref) eq 'HASH') {
+                %{$hashref} = (
+                                sums     => \%chksum,
+                                versions => \%revnum,
+                              );
+            }
+        }
+    }
     return;
 }
+
+sub fetch_dns_checksums {
+    my %checksums;
+    my $machine_dom = &Apache::lonnet::host_domain($perlvar{'lonHostID'});
+    my $loncaparev = &get_server_loncaparev($machine_dom,$perlvar{'lonHostID'});
+    my ($release,$timestamp) = split(/\-/,$loncaparev);
+    &get_dns("/adm/dns/checksums/$release",\&parse_dns_checksums_tab,1,1,
+             \%checksums);
+    return \%checksums;
+}
+
 # ------------------------------------------------------------ Read domain file
 {
     my $loaded;
@@ -10609,8 +13354,8 @@ sub get_dns {
     }
 
     sub load_domain_tab {
-	my ($ignore_cache) = @_;
-	&get_dns('/adm/dns/domain',\&parse_domain_tab,$ignore_cache);
+	my ($ignore_cache,$nocache) = @_;
+	&get_dns('/adm/dns/domain',\&parse_domain_tab,$ignore_cache,$nocache);
 	my $fh;
 	if (open($fh,"<".$perlvar{'lonTabDir'}.'/domain.tab')) {
 	    my @lines = <$fh>;
@@ -10688,6 +13433,7 @@ sub get_dns {
 	&purge_remembered();
 	&reset_domain_info();
 	&reset_hosts_ip_info();
+        undef(%internetdom);
 	undef(%name_to_host);
 	undef(%hostname);
 	undef(%hostdom);
@@ -10696,8 +13442,8 @@ sub get_dns {
     }
 
     sub load_hosts_tab {
-	my ($ignore_cache) = @_;
-	&get_dns('/adm/dns/hosts',\&parse_hosts_tab,$ignore_cache);
+	my ($ignore_cache,$nocache) = @_;
+	&get_dns('/adm/dns/hosts',\&parse_hosts_tab,$ignore_cache,$nocache);
 	open(my $config,"<$perlvar{'lonTabDir'}/hosts.tab");
 	my @config = <$config>;
 	&parse_hosts_tab(\@config);
@@ -10719,7 +13465,8 @@ sub get_dns {
     }
 
     sub all_names {
-	&load_hosts_tab() if (!$loaded);
+        my ($ignore_cache,$nocache) = @_;
+	&load_hosts_tab($ignore_cache,$nocache) if (!$loaded);
 
 	return %name_to_host;
     }
@@ -10729,6 +13476,11 @@ sub get_dns {
         return %hostdom;
     }
 
+    sub all_host_intdom {
+        &load_hosts_tab() if (!$loaded);
+        return %internetdom;
+    }
+
     sub is_library {
 	&load_hosts_tab() if (!$loaded);
 
@@ -10841,7 +13593,7 @@ sub get_dns {
     }
     
     sub get_iphost {
-	my ($ignore_cache) = @_;
+	my ($ignore_cache,$nocache) = @_;
 
 	if (!$ignore_cache) {
 	    if (%iphost) {
@@ -10865,7 +13617,7 @@ sub get_dns {
 	    %old_name_to_ip = %{$ip_info->[1]};
 	}
 
-	my %name_to_host = &all_names();
+	my %name_to_host = &all_names($ignore_cache,$nocache);
 	foreach my $name (keys(%name_to_host)) {
 	    my $ip;
 	    if (!exists($name_to_ip{$name})) {
@@ -10890,9 +13642,11 @@ sub get_dns {
 	    }
 	    push(@{$iphost{$ip}},@{$name_to_host{$name}});
 	}
-	&Apache::lonnet::do_cache_new('iphost','iphost',
-				      [\%iphost,\%name_to_ip,\%lonid_to_ip],
-				      48*60*60);
+        unless ($nocache) {
+	    &do_cache_new('iphost','iphost',
+		          [\%iphost,\%name_to_ip,\%lonid_to_ip],
+		          48*60*60);
+        }
 
 	return %iphost;
     }
@@ -10948,15 +13702,48 @@ sub get_dns {
             }
             $seen{$prim_ip} = 1;
         }
-        return &Apache::lonnet::do_cache_new('internetnames',$lonid,\@idns,12*60*60);
+        return &do_cache_new('internetnames',$lonid,\@idns,12*60*60);
     }
 
 }
 
 sub all_loncaparevs {
-    return qw(1.1 1.2 1.3 2.0 2.1 2.2 2.3 2.4 2.5 2.6 2.7 2.8 2.9 2.10);
+    return qw(1.1 1.2 1.3 2.0 2.1 2.2 2.3 2.4 2.5 2.6 2.7 2.8 2.9 2.10 2.11);
+}
+
+# ---------------------------------------------------------- Read loncaparev table
+{
+    sub load_loncaparevs { 
+        if (-e "$perlvar{'lonTabDir'}/loncaparevs.tab") {
+            if (open(my $config,"<$perlvar{'lonTabDir'}/loncaparevs.tab")) {
+                while (my $configline=<$config>) {
+                    chomp($configline);
+                    my ($hostid,$loncaparev)=split(/:/,$configline);
+                    $loncaparevs{$hostid}=$loncaparev;
+                }
+                close($config);
+            }
+        }
+    }
+}
+
+# ---------------------------------------------------------- Read serverhostID table
+{
+    sub load_serverhomeIDs {
+        if (-e "$perlvar{'lonTabDir'}/serverhomeIDs.tab") {
+            if (open(my $config,"<$perlvar{'lonTabDir'}/serverhomeIDs.tab")) {
+                while (my $configline=<$config>) {
+                    chomp($configline);
+                    my ($name,$id)=split(/:/,$configline);
+                    $serverhomeIDs{$name}=$id;
+                }
+                close($config);
+            }
+        }
+    }
 }
 
+
 BEGIN {
 
 # ----------------------------------- Read loncapa.conf and loncapa_apache.conf
@@ -11033,33 +13820,14 @@ BEGIN {
 }
 
 # ---------------------------------------------------------- Read loncaparev table
-{
-    if (-e "$perlvar{'lonTabDir'}/loncaparevs.tab") {
-        if (open(my $config,"<$perlvar{'lonTabDir'}/loncaparevs.tab")) {
-            while (my $configline=<$config>) {
-                chomp($configline);
-                my ($hostid,$loncaparev)=split(/:/,$configline);
-                $loncaparevs{$hostid}=$loncaparev;
-            }
-            close($config);
-        }
-    }
-}
+
+&load_loncaparevs();
 
 # ---------------------------------------------------------- Read serverhostID table
-{
-    if (-e "$perlvar{'lonTabDir'}/serverhomeIDs.tab") {
-        if (open(my $config,"<$perlvar{'lonTabDir'}/serverhomeIDs.tab")) {
-            while (my $configline=<$config>) {
-                chomp($configline);
-                my ($name,$id)=split(/:/,$configline);
-                $serverhomeIDs{$name}=$id;
-            }
-            close($config);
-        }
-    }
-}
 
+&load_serverhomeIDs();
+
+# ---------------------------------------------------------- Read releaseslist XML
 {
     my $file = $Apache::lonnet::perlvar{'lonTabDir'}.'/releaseslist.xml';
     if (-e $file) {
@@ -11069,7 +13837,15 @@ BEGIN {
                 my $item = $token->[1];
                 my $name = $token->[2]{'name'};
                 my $value = $token->[2]{'value'};
-                if ($item ne '' && $name ne '' && $value ne '') {
+                my $valuematch = $token->[2]{'valuematch'};
+                my $namematch = $token->[2]{'namematch'};
+                if ($item eq 'parameter') {
+                    if (($namematch ne '') || (($name ne '') && ($value ne '' || $valuematch ne ''))) {
+                        my $release = $parser->get_text();
+                        $release =~ s/(^\s*|\s*$ )//gx;
+                        $needsrelease{$item.':'.$name.':'.$value.':'.$valuematch.':'.$namematch} = $release;
+                    }
+                } elsif ($item ne '' && $name ne '') {
                     my $release = $parser->get_text();
                     $release =~ s/(^\s*|\s*$ )//gx;
                     $needsrelease{$item.':'.$name.':'.$value} = $release;
@@ -11258,8 +14034,8 @@ were new keys. I.E. 1:foo will become 1:
 
 Calling convention:
 
- my %record=&Apache::lonnet::restore($symb,$courseid,$domain,$uname,$home);
- &Apache::lonnet::cstore(\%newrecord,$symb,$courseid,$domain,$uname,$home);
+ my %record=&Apache::lonnet::restore($symb,$courseid,$domain,$uname);
+ &Apache::lonnet::cstore(\%newrecord,$symb,$courseid,$domain,$uname,$laststore);
 
 For more detailed information, see lonnet specific documentation.
 
@@ -11342,10 +14118,12 @@ the answer, and also caches if there is
 
 =item *
 X<idget()>
-B<idget($udom,@ids)>: find the usernames behind a list of IDs
-(IDs are a unique resource in a domain, there must be only 1 ID per
-username, and only 1 username per ID in a specific domain) (returns
-hash: id=>name,id=>name)
+B<idget($udom,$idsref,$namespace)>: find the usernames behind either 
+a list of student/employee IDs or clicker IDs
+(student/employee IDs are a unique resource in a domain, there must be 
+only 1 ID per username, and only 1 username per ID in a specific domain).
+clickerIDs are not necessarily unique, as students might share clickers.
+(returns hash: id=>name,id=>name)
 
 =item *
 X<idrget()>
@@ -11354,11 +14132,37 @@ usernames (returns hash: name=>id,name=>
 
 =item *
 X<idput()>
-B<idput($udom,%ids)>: store away a list of names and associated IDs
+B<idput($udom,$idsref,$uhome,$namespace)>: store away a list of 
+names and associated student/employee IDs or clicker IDs.
+
+=item *
+X<iddel()>
+B<iddel($udom,$idshashref,$uhome,$namespace)>: delete unwanted 
+student/employee ID or clicker ID username look-ups from domain.
+The homeserver ($uhome) and namespace ($namespace) are optional.
+If no $uhome is provided, it will be determined usig &homeserver()
+for each user.  If no $namespace is provided, the default is ids.
+
+=item *
+X<updateclickers()>
+B<updateclickers($udom,$action,$idshashref,$uhome,$critical)>: update 
+clicker ID-to-username look-ups in clickers.db on library server.
+Permitted actions are add or del (i.e., add or delete). The 
+clickers.db contains clickerID as keys (escaped), and each corresponding
+value is an escaped comma-separated list of usernames (for whom the
+library server is the homeserver), who registered that particular ID.
+If $critical is true, the update will be sent via &critical, otherwise
+&reply() will be used.
 
 =item *
 X<rolesinit()>
-B<rolesinit($udom,$username,$authhost)>: get user privileges
+B<rolesinit($udom,$username)>: get user privileges.
+returns user role, first access and timer interval hashes
+
+=item *
+X<privileged()>
+B<privileged($username,$domain)>: returns a true if user has a
+privileged and active role (i.e. su or dc), false otherwise.
 
 =item *
 X<getsection()>
@@ -11389,19 +14193,51 @@ escaped strings of the action recorded i
 
 =item *
 
-allowed($priv,$uri,$symb,$role) : check for a user privilege; returns codes for allowed actions
+allowed($priv,$uri,$symb,$role,$clientip,$noblockcheck) : check for a user privilege; 
+returns codes for allowed actions.
+
+The first argument is required, all others are optional.
+
+$priv is the privilege being checked.
+$uri contains additional information about what is being checked for access (e.g.,
+URL, course ID etc.). 
+$symb is the unique resource instance identifier in a course; if needed,
+but not provided, it will be retrieved via a call to &symbread(). 
+$role is the role for which a priv is being checked (only used if priv is evb). 
+$clientip is the user's IP address (only used when checking for access to portfolio 
+files).
+$noblockcheck, if true, skips calls to &has_comm_blocking() for the bre priv. This 
+prevents recursive calls to &allowed.
+
  F: full access
  U,I,K: authentication modes (cxx only)
  '': forbidden
  1: user needs to choose course
  2: browse allowed
  A: passphrase authentication needed
+ B: access temporarily blocked because of a blocking event in a course.
 
 =item *
 
-definerole($rolename,$sysrole,$domrole,$courole) : define role; define a custom
-role rolename set privileges in format of lonTabs/roles.tab for system, domain,
-and course level
+constructaccess($url,$setpriv) : check for access to construction space URL
+
+See if the owner domain and name in the URL match those in the
+expected environment.  If so, return three element list
+($ownername,$ownerdomain,$ownerhome).
+
+Otherwise return the null string.
+
+If second argument 'setpriv' is true, it assigns the privileges,
+and returns the same three element list, unless the owner has
+blocked "ad hoc" Domain Coordinator access to the Author Space,
+in which case the null string is returned.
+
+=item *
+
+definerole($rolename,$sysrole,$domrole,$courole,$uname,$udom) : define role;
+define a custom role rolename set privileges in format of lonTabs/roles.tab
+for system, domain, and course level. $uname and $udom are optional (current
+user's username and domain will be used when either of $uname or $udom are absent.
 
 =item *
 
@@ -11415,7 +14251,7 @@ environment).  If no custom name is defi
    
 =item *
 
-get_my_roles($uname,$udom,$context,$types,$roles,$roledoms,$withsec) :
+get_my_roles($uname,$udom,$context,$types,$roles,$roledoms,$withsec,$hidepriv) :
 All arguments are optional. Returns a hash of a roles, either for
 co-author/assistant author roles for a user's Construction Space
 (default), or if $context is 'userroles', roles for the user himself,
@@ -11429,6 +14265,41 @@ of role statuses (active, future or prev
 to restrict the list of roles reported. If no array ref is 
 provided for types, will default to return only active roles.
 
+=item *
+
+in_course($udom,$uname,$cdom,$cnum,$type,$hideprivileged) : determine if
+user: $uname:$udom has a role in the course: $cdom_$cnum. 
+
+Additional optional arguments are: $type (if role checking is to be restricted 
+to certain user status types -- previous (expired roles), active (currently
+available roles) or future (roles available in the future), and
+$hideprivileged -- if true will not report course roles for users who
+have active Domain Coordinator role in course's domain or in additional
+domains (specified in 'Domains to check for privileged users' in course
+environment -- set via:  Course Settings -> Classlists and staff listing).
+
+=item *
+
+privileged($username,$domain,$possdomains,$possroles) : returns 1 if user
+$username:$domain is a privileged user (e.g., Domain Coordinator or Super User)
+$possdomains and $possroles are optional array refs -- to domains to check and
+roles to check.  If $possdomains is not specified, a dump will be done of the
+users' roles.db to check for a dc or su role in any domain. This can be
+time consuming if &privileged is called repeatedly (e.g., when displaying a
+classlist), so in such cases, supplying a $possdomains array is preferred, as
+this then allows &privileged_by_domain() to be used, which caches the identity
+of privileged users, eliminating the need for repeated calls to &dump().
+
+=item *
+
+privileged_by_domain($possdomains,$roles) : returns a hash of a hash of a hash,
+where the outer hash keys are domains specified in the $possdomains array ref,
+next inner hash keys are privileged roles specified in the $roles array ref,
+and the innermost hash contains key = value pairs for username:domain = end:start
+for active or future "privileged" users with that role in that domain. To avoid
+repeated dumps of domain roles -- via &get_domain_roles() -- contents of the
+innerhash are cached using priv_$role and $dom as the identifiers.
+
 =back
 
 =head2 User Modification
@@ -11470,8 +14341,8 @@ or when Autoupdate.pl is run by cron in
 modifystudent
 
 modify a student's enrollment and identification information.
-The course id is resolved based on the current users environment.  
-This means the envoking user must be a course coordinator or otherwise
+The course id is resolved based on the current user's environment.  
+This means the invoking user must be a course coordinator or otherwise
 associated with a course.
 
 This call is essentially a wrapper for lonnet::modifyuser and
@@ -11521,7 +14392,9 @@ Inputs:
 
 =item B<$context> role change context (shown in User Management Logs display in a course)
 
-=item B<$inststatus> institutional status of user - : separated string of escaped status types  
+=item B<$inststatus> institutional status of user - : separated string of escaped status types
+
+=item B<$credits> Number of credits student will earn from this class - only needs to be supplied if value needs to be different from default credits for class.
 
 =back
 
@@ -11529,20 +14402,20 @@ Inputs:
 
 modify_student_enrollment
 
-Change a students enrollment status in a class.  The environment variable
+Change a student's enrollment status in a class.  The environment variable
 'role.request.course' must be defined for this function to proceed.
 
 Inputs:
 
 =over 4
 
-=item $udom, students domain
+=item $udom, student's domain
 
-=item $uname, students name
+=item $uname, student's name
 
-=item $uid, students user id
+=item $uid, student's user id
 
-=item $first, students first name
+=item $first, student's first name
 
 =item $middle
 
@@ -11566,6 +14439,10 @@ Inputs:
 
 =item $context
 
+=item $credits, number of credits student will earn from this class
+
+=item $instsec, institutional course section code for student
+
 =back
 
 
@@ -11622,7 +14499,7 @@ If defined, the supplied username is use
 resdata($name,$domain,$type,@which) : request for current parameter
 setting for a specific $type, where $type is either 'course' or 'user',
 @what should be a list of parameters to ask about. This routine caches
-answers for 5 minutes.
+answers for 10 minutes.
 
 =item *
 
@@ -11631,6 +14508,9 @@ data base, returning a hash that is keye
 values that are the resource value.  I believe that the timestamps and
 versions are also returned.
 
+get_numsuppfiles($cnum,$cdom) : retrieve number of files in a course's
+supplemental content area. This routine caches the number of files for 
+10 minutes.
 
 =back
 
@@ -11651,6 +14531,19 @@ createcourse($udom,$description,$url,$co
 
 generate_coursenum($udom,$crstype) : get a unique (unused) course number in domain $udom for course type $crstype (Course or Community).
 
+=item *
+
+is_course($courseid), is_course($cdom, $cnum)
+
+Accepts either a combined $courseid (in the form of domain_courseid) or the
+two component version $cdom, $cnum. It checks if the specified course exists.
+
+Returns:
+    undef if the course doesn't exist, otherwise
+    in scalar context the combined courseid.
+    in list context the two components of the course identifier, domain and 
+    courseid.    
+
 =back
 
 =head2 Resource Subroutines
@@ -11678,10 +14571,15 @@ resource. Expects the local filesystem p
 
 =item *
 
-EXT($varname,$symb,$udom,$uname) : evaluates and returns the value of
-a vairety of different possible values, $varname should be a request
-string, and the other parameters can be used to specify who and what
-one is asking about.
+EXT($varname,$symb,$udom,$uname,$usection,$recurse,$cid) : evaluates 
+and returns the value of a variety of different possible values,
+$varname should be a request string, and the other parameters can be
+used to specify who and what one is asking about. Ordinarily, $cid 
+does not need to be specified, as it is retrived from 
+$env{'request.course.id'}, but &Apache::lonnet::EXT() is called
+within lonuserstate::loadmap() when initializing a course, before
+$env{'request.course.id'} has been set, so it needs to be provided
+in that one case.
 
 Possible values for $varname are environment.lastname (or other item
 from the envirnment hash), user.name (or someother aspect about the
@@ -11714,17 +14612,32 @@ will be stored for query
 
 =item *
 
-symbread($filename) : return symbolic list entry (filename argument optional);
+symbread($filename,$donotrecurse,$ignorecachednull,$checkforblock,$possibles) : 
+return symbolic list entry (all arguments optional). 
+
+Args: filename is the filename (including path) for the file for which a symb 
+is required; donotrecurse, if true will prevent calls to allowed() being made 
+to check access status if more than one resource was found in the bighash 
+(see rev. 1.249) to avoid an infinite loop if an ambiguous resource is part of 
+a randompick); ignorecachednull, if true will prevent a symb of '' being 
+returned if $env{$cache_str} is defined as ''; checkforblock if true will
+cause possible symbs to be checked to determine if they are subject to content
+blocking, if so they will not be included as possible symbs; possibles is a
+ref to a hash, which, as a side effect, will be populated with all possible 
+symbs (content blocking not tested).
+ 
 returns the data handle
 
 =item *
 
-symbverify($symb,$thisfn) : verifies that $symb actually exists and is
-a possible symb for the URL in $thisfn, and if is an encryypted
+symbverify($symb,$thisfn,$encstate) : verifies that $symb actually exists
+and is a possible symb for the URL in $thisfn, and if is an encrypted
 resource that the user accessed using /enc/ returns a 1 on success, 0
-on failure, user must be in a course, as it assumes the existance of
-the course initial hash, and uses $env('request.course.id'}
-
+on failure, user must be in a course, as it assumes the existence of
+the course initial hash, and uses $env('request.course.id'}.  The third
+arg is an optional reference to a scalar.  If this arg is passed in the 
+call to symbverify, it will be set to 1 if the symb has been set to be 
+encrypted; otherwise it will be null.  
 
 =item *
 
@@ -11777,6 +14690,34 @@ expirespread($uname,$udom,$stype,$usymb)
 devalidate($symb) : devalidate temporary spreadsheet calculations,
 forcing spreadsheet to reevaluate the resource scores next time.
 
+=item * 
+
+can_edit_resource($file,$cnum,$cdom,$resurl,$symb,$group) : determine if current user can edit a particular resource,
+when viewing in course context.
+
+ input: six args -- filename (decluttered), course number, course domain,
+                    url, symb (if registered) and group (if this is a 
+                    group item -- e.g., bulletin board, group page etc.).
+
+ output: array of five scalars --
+         $cfile -- url for file editing if editable on current server
+         $home -- homeserver of resource (i.e., for author if published,
+                                          or course if uploaded.).
+         $switchserver --  1 if server switch will be needed.
+         $forceedit -- 1 if icon/link should be to go to edit mode 
+         $forceview -- 1 if icon/link should be to go to view mode
+
+=item *
+
+is_course_upload($file,$cnum,$cdom)
+
+Used in course context to determine if current file was uploaded to 
+the course (i.e., would be found in /userfiles/docs on the course's 
+homeserver.
+
+  input: 3 args -- filename (decluttered), course number and course domain.
+  output: boolean -- 1 if file was uploaded.
+
 =back
 
 =head2 Storing/Retreiving Data
@@ -11785,15 +14726,21 @@ forcing spreadsheet to reevaluate the re
 
 =item *
 
-store($storehash,$symb,$namespace,$udom,$uname) : stores hash permanently
-for this url; hashref needs to be given and should be a \%hashname; the
-remaining args aren't required and if they aren't passed or are '' they will
-be derived from the env
+store($storehash,$symb,$namespace,$udom,$uname,$laststore) : stores hash
+permanently for this url; hashref needs to be given and should be a \%hashname;
+the remaining args aren't required and if they aren't passed or are '' they will
+be derived from the env (with the exception of $laststore, which is an 
+optional arg used when a user's submission is stored in grading).
+$laststore is $version=$timestamp, where $version is the most recent version
+number retrieved for the corresponding $symb in the $namespace db file, and
+$timestamp is the timestamp for that transaction (UNIX time).
+$laststore is currently only passed when cstore() is called by 
+structuretags::finalize_storage().
 
 =item *
 
-cstore($storehash,$symb,$namespace,$udom,$uname) : same as store but
-uses critical subroutine
+cstore($storehash,$symb,$namespace,$udom,$uname,$laststore) : same as store
+but uses critical subroutine
 
 =item *
 
@@ -11816,10 +14763,11 @@ $range should be either an integer '100'
 
 =item *
 
-putstore($namespace,$symb,$version,$storehash,$udomain,$uname) :
+putstore($namespace,$symb,$version,$storehash,$udomain,$uname,$tolog) :
 replaces a &store() version of data with a replacement set of data
 for a particular resource in a namespace passed in the $storehash hash 
-reference
+reference. If $tolog is true, the transaction is logged in the courselog
+with an action=PUTSTORE.
 
 =item *
 
@@ -11929,15 +14877,91 @@ server ($udom and $uhome are optional)
 
 =item * 
 
-get_domain_defaults($target_domain) : returns hash with defaults for
-authentication and language in the domain. Keys are: auth_def, auth_arg_def,
-lang_def; corresponsing values are authentication type (internal, krb4, krb5,
-or localauth), initial password or a kerberos realm, language (e.g., en-us).
-Values are retrieved from cache (if current), or from domain's configuration.db
-(if available), or lastly from values in lonTabs/dns_domain,tab, 
-or lonTabs/domain.tab. 
+get_domain_defaults($target_domain,$ignore_cache) : returns hash with defaults 
+for: authentication, language, quotas, timezone, date locale, and portal URL in
+the target domain.
+
+May also include additional key => value pairs for the following groups:
+
+=over
+
+=item
+disk quotas (MB allocated by default to portfolios and authoring spaces).
 
-%domdefaults = &get_auth_defaults($target_domain);
+=over
+
+=item defaultquota, authorquota
+
+=back
+
+=item
+tools (availability of aboutme page, blog, webDAV access for authoring spaces,
+portfolio for users).
+
+=over
+
+=item
+aboutme, blog, webdav, portfolio
+
+=back
+
+=item
+requestcourses: ability to request courses, and how requests are processed.
+
+=over
+
+=item
+official, unofficial, community, textbook, placement
+
+=back
+
+=item
+inststatus: types of institutional affiliation, and order in which they are displayed.
+
+=over
+
+=item
+inststatustypes, inststatusorder, inststatusguest
+
+=back
+
+=item
+coursedefaults: can PDF forms can be created, default credits for courses, default quotas (MB)
+for course's uploaded content.
+
+=over
+
+=item
+canuse_pdfforms, officialcredits, unofficialcredits, textbookcredits, officialquota, unofficialquota, 
+communityquota, textbookquota, placementquota
+
+=back
+
+=item
+usersessions: set options for hosting of your users in other domains, and hosting of users from other domains
+on your servers.
+
+=over
+
+=item 
+remotesessions, hostedsessions
+
+=back
+
+=back
+
+In cases where a domain coordinator has never used the "Set Domain Configuration"
+utility to create a configuration.db file on a domain's primary library server 
+only the following domain defaults: auth_def, auth_arg_def, lang_def
+-- corresponding values are authentication type (internal, krb4, krb5,
+or localauth), initial password or a kerberos realm, language (e.g., en-us) -- 
+will be available. Values are retrieved from cache (if current), unless the
+optional $ignore_cache arg is true, or from domain's configuration.db (if available),
+or lastly from values in lonTabs/dns_domain,tab, or lonTabs/domain.tab.
+
+Typical usage:
+
+%domdefaults = &get_domain_defaults($target_domain);
 
 =back
 
@@ -12162,7 +15186,8 @@ filelocation except for hrefs
 
 =item *
 
-declutter() : declutters URLs (remove docroot, beginning slashes, 'res' etc)
+declutter() : declutters URLs -- remove beginning slashes, 'res' etc.
+also removes beginning /home/httpd/html unless /priv/ follows it.
 
 =back
 
@@ -12332,6 +15357,8 @@ Internal notes:
  
  Locks on files (resulting from submission of portfolio file to a homework problem stored in array of arrays.
 
+=item *
+
 modify_access_controls():
 
 Modifies access controls for a portfolio file
@@ -12349,7 +15376,51 @@ Returns:
 3. reference to hash of any new or updated access controls.
 4. reference to hash used to map incoming IDs to uniqueIDs assigned to control.
    key = integer (inbound ID)
-   value = uniqueID  
+   value = uniqueID
+
+=item *
+
+get_timebased_id():
+
+Attempts to get a unique timestamp-based suffix for use with items added to a 
+course via the Course Editor (e.g., folders, composite pages, 
+group bulletin boards).
+
+Args: (first three required; six others optional)
+
+1. prefix (alphanumeric): of keys in hash, e.g., suppsequence, docspage,
+   docssequence, or name of group
+
+2. keyid (alphanumeric): name of temporary locking key in hash,
+   e.g., num, boardids
+
+3. namespace: name of gdbm file used to store suffixes already assigned;  
+   file will be named nohist_namespace.db
+
+4. cdom: domain of course; default is current course domain from %env
+
+5. cnum: course number; default is current course number from %env
+
+6. idtype: set to concat if an additional digit is to be appended to the 
+   unix timestamp to form the suffix, if the plain timestamp is already
+   in use.  Default is to not do this, but simply increment the unix 
+   timestamp by 1 until a unique key is obtained.
+
+7. who: holder of locking key; defaults to user:domain for user.
+
+8. locktries: number of attempts to obtain a lock (sleep of 1s before 
+   retrying); default is 3.
+
+9. maxtries: number of attempts to obtain a unique suffix; default is 20.  
+
+Returns:
+
+1. suffix obtained (numeric)
+
+2. result of deleting locking key (ok if deleted, or lock never obtained)
+
+3. error: contains (localized) error message if an error occurred.
+
 
 =back